[llvm] [AtomicExpand] Add elementwise expansion (PR #196464)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 23:22:10 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/196464
>From 337da612f436dfb74189fa710137c7c90be3d46a Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 11 May 2026 20:07:15 +0000
Subject: [PATCH 01/15] initial commit
---
llvm/include/llvm/CodeGen/TargetLowering.h | 30 +-
llvm/lib/CodeGen/AtomicExpandPass.cpp | 162 +++++++++-
.../NVPTX/expand-atomic-rmw-elementwise.ll | 285 ++++++++++++++++++
.../X86/expand-atomic-rmw-elementwise.ll | 72 +++++
4 files changed, 541 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9cab67b6e657a..057fe9d8da075 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -270,6 +270,20 @@ class LLVM_ABI TargetLoweringBase {
Expand, // Generic expansion in terms of other atomic operations.
CustomExpand, // Custom target-specific expansion using TLI hooks.
+ // Halve an elementwise vector atomicrmw and re-ask the target how to expand
+ // each half. Split an `atomicrmw elementwise <N x T>` into either
+ // two `atomicrmw elementwise <N/2 x T>` (when N > 2) or two scalar
+ // `atomicrmw T` (when N == 2). An `<1 x T>` elementwise RMW is collapsed
+ // directly to a scalar `atomicrmw T`. Each resulting atomicrmw is fed back
+ // through the expansion pipeline, so the target's
+ // `shouldExpandAtomicRMWInIR` is re-queried at the halved width and may
+ // return any expansion kind there:
+ // - `None` to preserve at that width (e.g. a native vector atomic),
+ // - `Elementwise` to keep halving,
+ // - `CmpXChg` (or any other kind) to commit to that expansion for the
+ // current width.
+ Elementwise,
+
// Rewrite to a non-atomic form for use in a known non-preemptible
// environment.
NotAtomic
@@ -2496,10 +2510,18 @@ class LLVM_ABI TargetLoweringBase {
/// AtomicExpand pass.
virtual AtomicExpansionKind
shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
- if (RMWI->getOperation() == AtomicRMWInst::Xchg &&
- (RMWI->getValOperand()->getType()->isFloatingPointTy() ||
- RMWI->getValOperand()->getType()->isPointerTy()))
- return AtomicExpansionKind::CastToInteger;
+ if (RMWI->getOperation() == AtomicRMWInst::Xchg) {
+ Type *Ty = RMWI->getValOperand()->getType();
+ if (Ty->isFloatingPointTy() || Ty->isPointerTy())
+ return AtomicExpansionKind::CastToInteger;
+
+ if (Ty->isVectorTy()) {
+ TypeSize SizeInBits = RMWI->getDataLayout().getTypeStoreSizeInBits(Ty);
+ if (!SizeInBits.isScalable() &&
+ SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CastToInteger;
+ }
+ }
return AtomicExpansionKind::None;
}
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 059c0d5cb7b8b..1c6307dc8856f 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -44,6 +44,7 @@
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@@ -105,6 +106,7 @@ class AtomicExpandImpl {
bool tryExpandAtomicStore(StoreInst *SI);
void expandAtomicStoreToXChg(StoreInst *SI);
bool tryExpandAtomicRMW(AtomicRMWInst *AI);
+ void expandElementwiseAtomicRMW(AtomicRMWInst *AI);
AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
Value *
insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
@@ -382,6 +384,24 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
+ if (RMWI->isElementwise()) {
+ auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
+ if (ExpansionKind ==
+ TargetLoweringBase::AtomicExpansionKind::Elementwise) {
+ // If the target wants fence-based lowering for this elementwise RMW,
+ // insert the fences before splitting and downgrade the RMW ordering
+ // first. The split operations inherit the downgraded ordering, so they
+ // do not each get their own fence pair.
+ tryInsertFencesForAtomic(
+ RMWI,
+ isReleaseOrStronger(RMWI->getOrdering()) ||
+ isAcquireOrStronger(RMWI->getOrdering()),
+ TLI->atomicOperationOrderAfterFenceSplit(RMWI));
+ expandElementwiseAtomicRMW(RMWI);
+ return true;
+ }
+ }
+
if (!atomicSizeSupported(TLI, RMWI)) {
expandAtomicRMWToLibcall(RMWI);
return true;
@@ -608,6 +628,125 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
return NewRMWI;
}
+/// Halve an elementwise vector atomicrmw and feed each half back through the
+/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
+/// is re-queried at the halved width and may return any expansion kind there
+/// (e.g. None to preserve as a native vector atomic, Elementwise to keep
+/// halving, CmpXChg to emit one wide cmpxchg loop at the halved width). Three
+/// cases:
+/// N == 1: collapse directly to one scalar atomicrmw T
+/// N == 2: split into two scalar atomicrmw T (low at Ptr, high at
+/// gep inbounds T, Ptr, 1); reassemble via two insertelement's.
+/// N > 2: split into two atomicrmw elementwise <N/2 x T> (low at Ptr, high
+/// at gep inbounds <N/2 x T>, Ptr, 1); reassemble via a
+/// shufflevector.
+/// Note that halving works because the vectors must always be a power of two.
+///
+/// Each new atomicrmw inherits the original ordering/syncscope/volatility and
+/// metadata, and is fed back through processAtomicInstr() so further expansion
+/// fires per-half.
+void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
+ assert(AI->isElementwise());
+ auto *VecTy = cast<FixedVectorType>(AI->getType());
+ Type *LaneTy = VecTy->getElementType();
+ const unsigned NumLanes = VecTy->getNumElements();
+ Value *Ptr = AI->getPointerOperand();
+ Value *Val = AI->getValOperand();
+
+ ReplacementIRBuilder Builder(AI, *DL);
+ IntegerType *IdxTy =
+ DL->getIndexType(AI->getContext(), AI->getPointerAddressSpace());
+
+ auto CreateRMWInstruction = [&](Value *HalfPtr, Value *HalfVal, Align A,
+ bool Elementwise) -> AtomicRMWInst * {
+ auto *NewAI =
+ Builder.CreateAtomicRMW(AI->getOperation(), HalfPtr, HalfVal, A,
+ AI->getOrdering(), AI->getSyncScopeID());
+ NewAI->setVolatile(AI->isVolatile());
+ NewAI->setElementwise(Elementwise);
+ copyMetadataForAtomic(*NewAI, *AI);
+ return NewAI;
+ };
+
+ Constant *IdxZero = ConstantInt::get(IdxTy, 0);
+ Constant *IdxOne = ConstantInt::get(IdxTy, 1);
+
+ // N == 1: collapse to a single scalar atomicrmw T and wrap the result back
+ // into a <1 x T> vector.
+ if (NumLanes == 1) {
+ Value *LaneVal = Builder.CreateExtractElement(Val, IdxZero, "lane.val");
+ AtomicRMWInst *LaneRMW = CreateRMWInstruction(Ptr, LaneVal, AI->getAlign(),
+ /*Elementwise=*/false);
+ Value *Result = Builder.CreateInsertElement(PoisonValue::get(VecTy),
+ LaneRMW, IdxZero, "lane.old");
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LaneRMW);
+ return;
+ }
+
+ const uint64_t LaneBytes = DL->getTypeStoreSize(LaneTy).getFixedValue();
+
+ // N == 2 base case: split directly into two scalar atomicrmw T, one per
+ // lane.
+ if (NumLanes == 2) {
+ Value *LoVal = Builder.CreateExtractElement(Val, IdxZero, "lo.val");
+ Value *HiVal = Builder.CreateExtractElement(Val, IdxOne, "hi.val");
+ Value *HiPtr = Builder.CreateInBoundsGEP(LaneTy, Ptr, IdxOne, "hi.ptr");
+ Align LoAlign = AI->getAlign();
+ Align HiAlign = commonAlignment(LoAlign, LaneBytes);
+
+ AtomicRMWInst *LoRMW =
+ CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/false);
+ AtomicRMWInst *HiRMW =
+ CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/false);
+
+ Value *Result = PoisonValue::get(VecTy);
+ Result = Builder.CreateInsertElement(Result, LoRMW, IdxZero, "lo.old");
+ Result = Builder.CreateInsertElement(Result, HiRMW, IdxOne, "hi.old");
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LoRMW);
+ processAtomicInstr(HiRMW);
+ return;
+ }
+
+ // N > 2: split into two <N/2 x T> elementwise atomicrmws and recurse via
+ // processAtomicInstr().
+ assert(NumLanes % 2 == 0 &&
+ "elementwise atomicrmw vector length must be a power of two");
+ const unsigned HalfLanes = NumLanes / 2;
+ auto *HalfVecTy = FixedVectorType::get(LaneTy, HalfLanes);
+ const uint64_t HalfBytes = DL->getTypeStoreSize(HalfVecTy).getFixedValue();
+
+ SmallVector<int, 16> LoMask(HalfLanes), HiMask(HalfLanes);
+ for (unsigned I = 0; I < HalfLanes; ++I) {
+ LoMask[I] = static_cast<int>(I);
+ HiMask[I] = static_cast<int>(HalfLanes + I);
+ }
+ Value *LoVal = Builder.CreateShuffleVector(Val, LoMask, "lo.val");
+ Value *HiVal = Builder.CreateShuffleVector(Val, HiMask, "hi.val");
+
+ Value *HiPtr = Builder.CreateInBoundsGEP(HalfVecTy, Ptr, IdxOne, "hi.ptr");
+ Align LoAlign = AI->getAlign();
+ Align HiAlign = commonAlignment(LoAlign, HalfBytes);
+
+ AtomicRMWInst *LoRMW =
+ CreateRMWInstruction(Ptr, LoVal, LoAlign, /*Elementwise=*/true);
+ AtomicRMWInst *HiRMW =
+ CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
+
+ SmallVector<int, 16> Mask(NumLanes);
+ for (unsigned I = 0; I < NumLanes; ++I)
+ Mask[I] = static_cast<int>(I);
+ Value *Result = Builder.CreateShuffleVector(LoRMW, HiRMW, Mask, "old");
+
+ AI->replaceAllUsesWith(Result);
+ AI->eraseFromParent();
+ processAtomicInstr(LoRMW);
+ processAtomicInstr(HiRMW);
+}
+
bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {
switch (TLI->shouldExpandAtomicLoadInIR(LI)) {
case TargetLoweringBase::AtomicExpansionKind::None:
@@ -1037,6 +1176,13 @@ static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::Nand: {
+ if (PMV.ValueType->isVectorTy()) {
+ // Avoid letting packed integer arithmetic carry across vector lanes.
+ Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
+ Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc);
+ return insertMaskedValue(Builder, Loaded, NewVal, PMV);
+ }
+
// The other arithmetic ops need to be masked into place.
Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Shifted_Inc);
Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
@@ -1099,8 +1245,10 @@ void AtomicExpandImpl::expandPartwordAtomicRMW(
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
Value *ValOperand_Shifted = nullptr;
- if (Op == AtomicRMWInst::Xchg || Op == AtomicRMWInst::Add ||
- Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Nand) {
+ if (Op == AtomicRMWInst::Xchg ||
+ (!PMV.ValueType->isVectorTy() &&
+ (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub ||
+ Op == AtomicRMWInst::Nand))) {
Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
@@ -1143,9 +1291,15 @@ AtomicRMWInst *AtomicExpandImpl::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
+ Value *ValOp = AI->getValOperand();
+ if (ValOp->getType()->isVectorTy())
+ // For vectors, bitcast to the integer type before extending. Note that
+ // or/xor/and on vectors are equivalent to the same operation on an integer
+ // that spans the vector, so we can use the integer type for the operation.
+ ValOp = Builder.CreateBitCast(ValOp, PMV.IntValueType);
Value *ValOperand_Shifted =
- Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
- PMV.ShiftAmt, "ValOperand_Shifted");
+ Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
+ "ValOperand_Shifted");
Value *NewOperand;
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..b573a11f1e4e2
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,285 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=nvptx64-nvidia-cuda -mcpu=sm_90 -mattr=+ptx83 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW and wraps
+; the old value back into a single-lane vector.
+define <1 x float> @fadd_v1f32_elementwise(ptr %addr, <1 x float> %val) {
+; CHECK-LABEL: @fadd_v1f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LANE_VAL:%.*]] = extractelement <1 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR:%.*]], float [[LANE_VAL]] monotonic, align 4
+; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: ret <1 x float> [[LANE_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise fadd ptr %addr, <1 x float> %val monotonic
+ ret <1 x float> %old
+}
+
+; Elementwise expansion: scalar f32 atomics are legal, so the vector is split
+; into scalar atomicrmw instructions.
+define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: @fadd_v2f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LO_VAL]] monotonic, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP1]], i64 1
+; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise fadd ptr %addr, <2 x float> %val monotonic
+ ret <2 x float> %old
+}
+
+; Metadata attached to an elementwise RMW is preserved on each split child.
+define <2 x i32> @add_metadata_v2i32_elementwise(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: @add_metadata_v2i32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x i32> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x i32> [[VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds i32, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL]] monotonic, align 8, !mmra [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[HI_VAL]] monotonic, align 4, !mmra [[META0]]
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x i32> [[LO_OLD]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: ret <2 x i32> [[HI_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <2 x i32> %val monotonic, !mmra !0
+ ret <2 x i32> %old
+}
+
+; Volatile is preserved while splitting elementwise operations.
+define <2 x float> @fadd_volatile_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: @fadd_volatile_v2f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw volatile fadd ptr [[ADDR]], float [[LO_VAL]] monotonic, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile fadd ptr [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP1]], i64 1
+; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
+;
+entry:
+ %old = atomicrmw volatile elementwise fadd ptr %addr, <2 x float> %val monotonic
+ ret <2 x float> %old
+}
+
+; Ordered elementwise expansion: fences should be inserted around the whole
+; split expansion rather than around each split child.
+define <4 x i32> @add_acq_rel_v4i32_elementwise(ptr %addr, <4 x i32> %val) {
+; CHECK-LABEL: @add_acq_rel_v4i32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: fence syncscope("block") release
+; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <4 x i32> [[VAL:%.*]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <4 x i32> [[VAL]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <2 x i32>, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[LO_VAL2:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 0
+; CHECK-NEXT: [[HI_VAL3:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds i32, ptr [[ADDR]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL2]] syncscope("block") monotonic, align 16
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[HI_PTR4]], i32 [[HI_VAL3]] syncscope("block") monotonic, align 4
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x i32> [[LO_OLD]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: [[LO_VAL5:%.*]] = extractelement <2 x i32> [[HI_VAL]], i64 0
+; CHECK-NEXT: [[HI_VAL6:%.*]] = extractelement <2 x i32> [[HI_VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR7:%.*]] = getelementptr inbounds i32, ptr [[HI_PTR]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[LO_VAL5]] syncscope("block") monotonic, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw add ptr [[HI_PTR7]], i32 [[HI_VAL6]] syncscope("block") monotonic, align 4
+; CHECK-NEXT: [[LO_OLD8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0
+; CHECK-NEXT: [[HI_OLD9:%.*]] = insertelement <2 x i32> [[LO_OLD8]], i32 [[TMP3]], i64 1
+; CHECK-NEXT: [[OLD1:%.*]] = shufflevector <2 x i32> [[HI_OLD]], <2 x i32> [[HI_OLD9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: fence syncscope("block") acquire
+; CHECK-NEXT: ret <4 x i32> [[OLD1]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <4 x i32> %val syncscope("block") acq_rel, align 16
+ ret <4 x i32> %old
+}
+
+; Seq-cst elementwise expansion also gets one outer fence before splitting.
+define <2 x i32> @add_seq_cst_v4i32_elementwise(ptr %addr, <2 x i32> %val) {
+; CHECK-LABEL: @add_seq_cst_v4i32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: fence syncscope("block") seq_cst
+; CHECK-NEXT: [[LO_VAL2:%.*]] = extractelement <2 x i32> [[LO_VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL3:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds i32, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL2]] syncscope("block") monotonic, align 16
+; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw add ptr [[HI_PTR4]], i32 [[HI_VAL3]] syncscope("block") monotonic, align 4
+; CHECK-NEXT: [[LO_OLD8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[HI_OLD9:%.*]] = insertelement <2 x i32> [[LO_OLD8]], i32 [[TMP3]], i64 1
+; CHECK-NEXT: fence syncscope("block") acquire
+; CHECK-NEXT: ret <2 x i32> [[HI_OLD9]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <2 x i32> %val syncscope("block") seq_cst, align 16
+ ret <2 x i32> %old
+}
+
+; Whole-vector cmpxchg expansion: sm_90 + ptx83 supports 128-bit cmpxchg, so
+; this vector can stay whole during AtomicExpand.
+define <8 x half> @fmin_v8f16_elementwise(ptr %addr, <8 x half> %val) {
+; CHECK-LABEL: @fmin_v8f16_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr [[ADDR:%.*]] monotonic, align 16
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i128 [[TMP0]] to <8 x half>
+; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CHECK: atomicrmw.start:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <8 x half> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x half> @llvm.minnum.v8f16(<8 x half> [[LOADED]], <8 x half> [[VAL:%.*]])
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x half> [[TMP2]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x half> [[LOADED]] to i128
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ADDR]], i128 [[TMP4]], i128 [[TMP3]] monotonic monotonic, align 16
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i128 [[NEWLOADED]] to <8 x half>
+; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK: atomicrmw.end:
+; CHECK-NEXT: ret <8 x half> [[TMP6]]
+;
+entry:
+ %old = atomicrmw elementwise fmin ptr %addr, <8 x half> %val monotonic, align 16
+ ret <8 x half> %old
+}
+
+; Vector xchg is bit-equivalent to xchg on the same-width integer type.
+define <4 x i8> @xchg_v4i8_elementwise(ptr %addr, <4 x i8> %val) {
+; CHECK-LABEL: @xchg_v4i8_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i8> [[VAL:%.*]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xchg ptr [[ADDR:%.*]], i32 [[TMP0]] monotonic, align 4
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to <4 x i8>
+; CHECK-NEXT: ret <4 x i8> [[TMP2]]
+;
+entry:
+ %old = atomicrmw elementwise xchg ptr %addr, <4 x i8> %val monotonic, align 4
+ ret <4 x i8> %old
+}
+
+; Partword bitwise vector atomics must bitcast before widening; zext directly
+; from <2 x i8> to i32 is invalid IR.
+define <2 x i8> @and_v2i8_partword_elementwise(ptr %addr, <2 x i8> %val) {
+; CHECK-LABEL: @and_v2i8_partword_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ADDR:%.*]], i64 -4)
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[ADDR]] to i64
+; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i8> [[VAL:%.*]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; CHECK-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; CHECK-NEXT: [[TMP4:%.*]] = atomicrmw and ptr [[ALIGNEDADDR]], i32 [[ANDOPERAND]] monotonic, align 4
+; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED]] to <2 x i8>
+; CHECK-NEXT: ret <2 x i8> [[TMP5]]
+;
+entry:
+ %old = atomicrmw elementwise and ptr %addr, <2 x i8> %val monotonic
+ ret <2 x i8> %old
+}
+
+; Partword vector arithmetic must operate lane-wise before reinserting into the
+; containing CAS word, otherwise packed integer carries can cross lanes.
+define <2 x i8> @add_v2i8_partword_elementwise(ptr %addr, <2 x i8> %val) {
+; CHECK-LABEL: @add_v2i8_partword_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ADDR:%.*]], i64 -4)
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[ADDR]] to i64
+; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = load atomic i32, ptr [[ALIGNEDADDR]] monotonic, align 4
+; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CHECK: atomicrmw.start:
+; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i16 [[EXTRACTED]] to <2 x i8>
+; CHECK-NEXT: [[NEW:%.*]] = add <2 x i8> [[TMP3]], [[VAL:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i8> [[NEW]] to i16
+; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK: atomicrmw.end:
+; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16 [[EXTRACTED3]] to <2 x i8>
+; CHECK-NEXT: ret <2 x i8> [[TMP6]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %addr, <2 x i8> %val monotonic
+ ret <2 x i8> %old
+}
+
+; Use two 128-bit cmpxchg loops to handle the 8 x i32 vector.
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; CHECK-LABEL: @nand_acq_rel_v8i32_global_cta(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: fence syncscope("block") release
+; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <8 x i32> [[VAL:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <8 x i32> [[VAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr addrspace(1) [[ADDR]] syncscope("block") monotonic, align 32
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i128 [[TMP0]] to <4 x i32>
+; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
+; CHECK: atomicrmw.start:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x i32> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[LOADED]], [[LO_VAL]]
+; CHECK-NEXT: [[NEW:%.*]] = xor <4 x i32> [[TMP2]], splat (i32 -1)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[NEW]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[LOADED]] to i128
+; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ADDR]], i128 [[TMP4]], i128 [[TMP3]] syncscope("block") monotonic monotonic, align 32
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i128, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP6]] = bitcast i128 [[NEWLOADED]] to <4 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK: atomicrmw.end:
+; CHECK-NEXT: [[TMP7:%.*]] = load atomic i128, ptr addrspace(1) [[HI_PTR]] syncscope("block") monotonic, align 16
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i128 [[TMP7]] to <4 x i32>
+; CHECK-NEXT: br label [[ATOMICRMW_START2:%.*]]
+; CHECK: atomicrmw.start2:
+; CHECK-NEXT: [[LOADED3:%.*]] = phi <4 x i32> [ [[TMP8]], [[ATOMICRMW_END]] ], [ [[TMP13:%.*]], [[ATOMICRMW_START2]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i32> [[LOADED3]], [[HI_VAL]]
+; CHECK-NEXT: [[NEW4:%.*]] = xor <4 x i32> [[TMP9]], splat (i32 -1)
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast <4 x i32> [[NEW4]] to i128
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast <4 x i32> [[LOADED3]] to i128
+; CHECK-NEXT: [[TMP12:%.*]] = cmpxchg ptr addrspace(1) [[HI_PTR]], i128 [[TMP11]], i128 [[TMP10]] syncscope("block") monotonic monotonic, align 16
+; CHECK-NEXT: [[SUCCESS5:%.*]] = extractvalue { i128, i1 } [[TMP12]], 1
+; CHECK-NEXT: [[NEWLOADED6:%.*]] = extractvalue { i128, i1 } [[TMP12]], 0
+; CHECK-NEXT: [[TMP13]] = bitcast i128 [[NEWLOADED6]] to <4 x i32>
+; CHECK-NEXT: br i1 [[SUCCESS5]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START2]]
+; CHECK: atomicrmw.end1:
+; CHECK-NEXT: [[OLD:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: fence syncscope("block") acquire
+; CHECK-NEXT: ret <8 x i32> [[OLD]]
+;
+entry:
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+!0 = !{!1, !2}
+!1 = !{!"foo", !"bar"}
+!2 = !{!"bux", !"baz"}
+
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..a19b5f0e0c1cd
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s
+
+; By default, elementwise atomicrmws are treated the same as the non-elementwise versions and are not expanded.
+; Targets in the future can take advantage of elementwise expansion by returning AtomicExpansionKind::Elementwise from shouldExpandAtomicRMWInIR.
+; Elementwise atomics work on integer vectors, but normal atomics don't.
+
+define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
+; CHECK-LABEL: define <4 x i32> @atomicrmw_add_elementwise(
+; CHECK-SAME: ptr [[P:%.*]], <4 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to i128
+; CHECK-NEXT: [[TMP2:%.*]] = call i128 @__atomic_fetch_add_16(ptr [[P]], i128 [[TMP1]], i32 0)
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i128 [[TMP2]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %old = atomicrmw elementwise add ptr %p, <4 x i32> %v monotonic
+ ret <4 x i32> %old
+}
+
+define <4 x float> @atomicrmw_fadd_elementwise(ptr %p, <4 x float> %v) {
+; CHECK-LABEL: define <4 x float> @atomicrmw_fadd_elementwise(
+; CHECK-SAME: ptr [[P:%.*]], <4 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <4 x float> [[LOADED]], [[V]]
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT: store <4 x float> [[LOADED]], ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[NEW]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i1 } poison, <4 x float> [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { <4 x float>, i1 } [[TMP7]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { <4 x float>, i1 } [[TMP7]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret <4 x float> [[NEWLOADED]]
+;
+ %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v monotonic
+ ret <4 x float> %old
+}
+
+define <4 x float> @atomicrmw_fadd(ptr %p, <4 x float> %v) {
+; CHECK-LABEL: define <4 x float> @atomicrmw_fadd(
+; CHECK-SAME: ptr [[P:%.*]], <4 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P]], align 16
+; CHECK-NEXT: br label %[[ATOMICRMW_START:.*]]
+; CHECK: [[ATOMICRMW_START]]:
+; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x float> [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT: [[NEW:%.*]] = fadd <4 x float> [[LOADED]], [[V]]
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT: store <4 x float> [[LOADED]], ptr [[TMP1]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float> [[NEW]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[P]], ptr [[TMP1]], i128 [[TMP3]], i32 0, i32 0)
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i1 } poison, <4 x float> [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { <4 x float>, i1 } [[TMP7]], 1
+; CHECK-NEXT: [[NEWLOADED]] = extractvalue { <4 x float>, i1 } [[TMP7]], 0
+; CHECK-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK: [[ATOMICRMW_END]]:
+; CHECK-NEXT: ret <4 x float> [[NEWLOADED]]
+;
+ %old = atomicrmw fadd ptr %p, <4 x float> %v monotonic
+ ret <4 x float> %old
+}
>From 3564ce1c8bb9cc3eed15aceb3d9549c5095eb0f0 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 11 May 2026 20:57:06 +0000
Subject: [PATCH 02/15] followup
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 123 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 11 +
.../NVPTX/atomicrmw-elementwise-sm60.ll | 19309 ++++++++++++++++
.../NVPTX/atomicrmw-elementwise-sm70.ll | 19233 +++++++++++++++
.../NVPTX/atomicrmw-elementwise-sm90.ll | 18160 +++++++++++++++
llvm/test/CodeGen/NVPTX/atomicrmw.py | 227 +-
6 files changed, 56944 insertions(+), 119 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 4047738a2d447..d1991d076d684 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7340,9 +7340,12 @@ void NVPTXTargetLowering::ReplaceNodeResults(
}
}
-NVPTXTargetLowering::AtomicExpansionKind
-NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
- Type *Ty = AI->getValOperand()->getType();
+/// Returns how NVPTX should expand a scalar atomicrmw with the given op and
+/// value type.
+static TargetLoweringBase::AtomicExpansionKind
+getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
+ const NVPTXSubtarget &STI) {
+ using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
// Try to lower LLVM atomicrmw fadd to PTX atomic.add. This is complicated
// by the weird FTZ behavior PTX atom.add has:
@@ -7392,19 +7395,22 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
if (AI->isFloatingPointOperation())
return AtomicExpansionKind::CmpXChg;
- assert(Ty->isIntegerTy() && "Ty should be integer at this point");
+ assert(Ty->isIntegerTy() &&
+ "Ty should be integer at this point. Integer operations must act on "
+ "an integer operand. We already cast non-integer CmpXChg operands to "
+ "integer.");
const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
- switch (AI->getOperation()) {
+ switch (Op) {
default:
return AtomicExpansionKind::CmpXChg;
- case AtomicRMWInst::BinOp::Xchg:
+ case AtomicRMWInst::Xchg:
if (BitWidth == 128)
return AtomicExpansionKind::None;
[[fallthrough]];
- case AtomicRMWInst::BinOp::And:
- case AtomicRMWInst::BinOp::Or:
- case AtomicRMWInst::BinOp::Xor:
+ case AtomicRMWInst::And:
+ case AtomicRMWInst::Or:
+ case AtomicRMWInst::Xor:
switch (BitWidth) {
case 8:
case 16:
@@ -7420,12 +7426,12 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
default:
llvm_unreachable("unsupported width encountered");
}
- case AtomicRMWInst::BinOp::Add:
- case AtomicRMWInst::BinOp::Sub:
- case AtomicRMWInst::BinOp::Max:
- case AtomicRMWInst::BinOp::Min:
- case AtomicRMWInst::BinOp::UMax:
- case AtomicRMWInst::BinOp::UMin:
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::Sub:
+ case AtomicRMWInst::Max:
+ case AtomicRMWInst::Min:
+ case AtomicRMWInst::UMax:
+ case AtomicRMWInst::UMin:
switch (BitWidth) {
case 8:
case 16:
@@ -7441,8 +7447,8 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
default:
llvm_unreachable("unsupported width encountered");
}
- case AtomicRMWInst::BinOp::UIncWrap:
- case AtomicRMWInst::BinOp::UDecWrap:
+ case AtomicRMWInst::UIncWrap:
+ case AtomicRMWInst::UDecWrap:
switch (BitWidth) {
case 32:
return AtomicExpansionKind::None;
@@ -7459,11 +7465,49 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
return AtomicExpansionKind::CmpXChg;
}
+NVPTXTargetLowering::AtomicExpansionKind
+NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
+ // TODO: once we support native elementwise vector atoms,
+ // return `AtomicExpansionKind::None` to preserve and lower them.
+ if (AI->isElementwise()) {
+ auto *VecTy = cast<FixedVectorType>(AI->getType());
+ Type *LaneTy = VecTy->getElementType();
+
+ // Collapse <1 x T> elementwise RMWs to scalar RMWs
+ if (VecTy->getNumElements() == 1)
+ return AtomicExpansionKind::Elementwise;
+
+ // If the scalar lane op is natively supported, return
+ // Elementwise so halving eventually bottoms out at the scalar base
+ // case, where this hook returns None for each scalar lane and the
+ // scalar `atom.*` instruction is preserved.
+ if (getScalarAtomicRMWExpansion(AI->getOperation(), LaneTy, STI) ==
+ AtomicExpansionKind::None)
+ return AtomicExpansionKind::Elementwise;
+
+ // If the whole vector fits a single native cmpxchg, emit one wide
+ // cmpxchg loop at the current width.
+ const DataLayout &DL = AI->getDataLayout();
+ uint64_t VecBits = DL.getTypeStoreSizeInBits(VecTy).getFixedValue();
+ if (VecBits <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CmpXChg;
+
+ // If the vector is too wide for a single native cmpxchg, halve further to
+ // emit multiple cmpxchg loops.
+ return AtomicExpansionKind::Elementwise;
+ }
+
+ return getScalarAtomicRMWExpansion(AI->getOperation(),
+ AI->getValOperand()->getType(), STI);
+}
+
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
- // This function returns true iff the operation is emulated using a CAS-loop,
- // or if it has the memory order seq_cst (which is not natively supported in
- // the PTX `atom` instruction).
+ // This function returns true iff AtomicExpandPass should enforce the
+ // instruction's ordering with fences instead of leaving that ordering on the
+ // atomic instruction itself. This covers CAS-loop emulation, seq_cst
+ // operations (which are not natively supported by the PTX `atom`
+ // instruction), and elementwise atomicrmw expansion.
//
// atomicrmw and cmpxchg instructions not efficiently supported by PTX
// are lowered to CAS emulation loops that preserve their memory order,
@@ -7471,8 +7515,13 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
// atom.cas.relaxed.sco instructions within the loop, and fences before and
// after the loop to restore order.
//
+ // For ordered elementwise atomicrmw, if we expand elementwise, it is also
+ // more efficient to insert fences around the whole split sequence once and
+ // downgrade the ordering for the entire sequence, rather than having the
+ // stronger ordering on every expanded atomicrmw.
+ //
// Atomic instructions efficiently supported by PTX are lowered to
- // `atom.<op>.<sem>.<scope` instruction with their corresponding memory order
+ // `atom.<op>.<sem>.<scope>` instruction with their corresponding memory order
// and scope. Since PTX does not support seq_cst, we emulate it by lowering to
// a fence.sc followed by an atom according to the PTX atomics ABI
// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
@@ -7480,9 +7529,12 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
return (cast<IntegerType>(CI->getCompareOperand()->getType())
->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
- if (auto *RI = dyn_cast<AtomicRMWInst>(I))
- return shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg ||
+ if (auto *RI = dyn_cast<AtomicRMWInst>(I)) {
+ AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
+ return Expansion == AtomicExpansionKind::CmpXChg ||
+ Expansion == AtomicExpansionKind::Elementwise ||
RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
+ }
return false;
}
@@ -7511,9 +7563,9 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=
STI.getMinCmpXchgSizeInBits())
return AtomicOrdering::Acquire;
- else if (auto *RI = dyn_cast<AtomicRMWInst>(I);
- RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
- shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
+ if (auto *RI = dyn_cast<AtomicRMWInst>(I);
+ RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
+ shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
return AtomicOrdering::Acquire;
return AtomicOrdering::Monotonic;
@@ -7558,12 +7610,21 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
auto SSID = getAtomicSyncScopeID(Inst);
assert(SSID.has_value() && "Expected an atomic operation");
- bool IsEmulated =
- CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
- : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
+ if (!isAcquireOrStronger(Ord))
+ return nullptr;
+
+ bool NeedsTrailingAcquireFence = false;
+ if (CI)
+ NeedsTrailingAcquireFence =
+ cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() <
+ STI.getMinCmpXchgSizeInBits();
+ else {
+ AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
+ NeedsTrailingAcquireFence = Expansion == AtomicExpansionKind::CmpXChg ||
+ Expansion == AtomicExpansionKind::Elementwise;
+ }
- if (isAcquireOrStronger(Ord) && IsEmulated)
+ if (NeedsTrailingAcquireFence)
return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
return nullptr;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index a7f0abc6eda79..2ec8994e0f1ed 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -146,6 +146,17 @@ class NVPTXTargetLowering : public TargetLowering {
bool isCheapToSpeculateCtlz(Type *Ty) const override { return true; }
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override {
+ Type *Ty = LI->getType();
+ // SelectionDAG can split vector computations, but not vector atomic loads.
+ // Splitting those loads into per-lane atomics would change the atomicity,
+ // so cast them to a single same-width integer atomic load instead.
+ if (Ty->isVectorTy()) {
+ TypeSize SizeInBits = LI->getDataLayout().getTypeStoreSizeInBits(Ty);
+ if (!SizeInBits.isScalable() &&
+ SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
+ return AtomicExpansionKind::CastToInteger;
+ }
+
return AtomicExpansionKind::None;
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
new file mode 100644
index 0000000000000..ab48ec6523719
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -0,0 +1,19309 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefix=SM60
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 255;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB0_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB1_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<14>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r9, %r8, %r1;
+; SM60-NEXT: not.b32 %r2, %r9;
+; SM60-NEXT: shl.b32 %r3, %r5, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r13, [%rd1];
+; SM60-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r13, %r2;
+; SM60-NEXT: or.b32 %r11, %r10, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM60-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM60-NEXT: mov.b32 %r13, %r4;
+; SM60-NEXT: @%p1 bra $L__BB4_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r12, %r4, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r12;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: cvt.u64.u32 %rd2, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r2;
+; SM60-NEXT: shl.b64 %rd4, %rd3, 32;
+; SM60-NEXT: or.b64 %rd5, %rd2, %rd4;
+; SM60-NEXT: cvt.u64.u32 %rd6, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: shl.b64 %rd8, %rd7, 32;
+; SM60-NEXT: or.b64 %rd9, %rd6, %rd8;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd10, [%rd1], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB16_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB17_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB18_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB18_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB19_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB19_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB20_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB21_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB21_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB22_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB22_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: add_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB23_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB23_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB23_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: add.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: add.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: add.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: add.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB23_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: add_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB32_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB33_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB33_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB34_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB34_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB35_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB35_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB36_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB37_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB38_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB39_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB39_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB39_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<4>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r2, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r3, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM60-NEXT: neg.s32 %r5, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r5, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM60-NEXT: neg.s32 %r7, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM60-NEXT: neg.s32 %r9, %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM60-NEXT: neg.s32 %r11, %r4;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: neg.s32 %r9, %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM60-NEXT: neg.s32 %r11, %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM60-NEXT: neg.s32 %r13, %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM60-NEXT: neg.s32 %r15, %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM60-NEXT: neg.s32 %r17, %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM60-NEXT: neg.s32 %r19, %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM60-NEXT: neg.s32 %r21, %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM60-NEXT: neg.s32 %r23, %r4;
+; SM60-NEXT: atom.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<5>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd3, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<8>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd4, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM60-NEXT: neg.s64 %rd6, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<14>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM60-NEXT: neg.s64 %rd8, %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM60-NEXT: neg.s64 %rd10, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM60-NEXT: neg.s64 %rd12, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<26>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: neg.s64 %rd10, %rd8;
+; SM60-NEXT: atom.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM60-NEXT: neg.s64 %rd12, %rd9;
+; SM60-NEXT: atom.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM60-NEXT: neg.s64 %rd14, %rd6;
+; SM60-NEXT: atom.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM60-NEXT: neg.s64 %rd16, %rd7;
+; SM60-NEXT: atom.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM60-NEXT: neg.s64 %rd18, %rd4;
+; SM60-NEXT: atom.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM60-NEXT: neg.s64 %rd20, %rd5;
+; SM60-NEXT: atom.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM60-NEXT: neg.s64 %rd22, %rd2;
+; SM60-NEXT: atom.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM60-NEXT: neg.s64 %rd24, %rd3;
+; SM60-NEXT: atom.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: mov.b32 %r5, 255;
+; SM60-NEXT: shl.b32 %r6, %r5, %r4;
+; SM60-NEXT: not.b32 %r7, %r6;
+; SM60-NEXT: shl.b32 %r8, %r1, %r4;
+; SM60-NEXT: or.b32 %r9, %r8, %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: mov.b32 %r4, 65535;
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: not.b32 %r6, %r5;
+; SM60-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r8, %r7, %r3;
+; SM60-NEXT: or.b32 %r9, %r8, %r6;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB50_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: and_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: and.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB51_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<12>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: mov.b32 %r5, 65535;
+; SM60-NEXT: shl.b32 %r6, %r5, %r4;
+; SM60-NEXT: not.b32 %r7, %r6;
+; SM60-NEXT: shl.b32 %r8, %r1, %r4;
+; SM60-NEXT: or.b32 %r9, %r8, %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM60-NEXT: shr.u32 %r11, %r10, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r11;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB53_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: and.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB54_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: and_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: and.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB55_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: and.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB55_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: and_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: and_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB64_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB65_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB66_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB66_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB67_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB67_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 65535;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB68_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB69_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB69_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB70_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB70_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB71_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r14, %r2;
+; SM60-NEXT: and.b32 %r6, %r13, %r1;
+; SM60-NEXT: xor.b32 %r7, %r6, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: xor.b32 %r8, %r5, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB71_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB71_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: and.b32 %r9, %r16, %r4;
+; SM60-NEXT: and.b32 %r10, %r15, %r3;
+; SM60-NEXT: xor.b32 %r11, %r10, -1;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: xor.b32 %r12, %r9, -1;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB71_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB72_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB74_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB74_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB75_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB75_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB75_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB75_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB75_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB75_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM60-NEXT: not.b64 %rd5, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB76_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd6, %rd10, %rd1;
+; SM60-NEXT: not.b64 %rd7, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB77_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd8, %rd11, %rd2;
+; SM60-NEXT: not.b64 %rd9, %rd8;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB77_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: and.b64 %rd10, %rd5, %rd3;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB78_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB78_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: and.b64 %rd12, %rd6, %rd4;
+; SM60-NEXT: not.b64 %rd13, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB78_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB78_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd14, %rd20, %rd1;
+; SM60-NEXT: not.b64 %rd15, %rd14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB78_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB78_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd16, %rd21, %rd2;
+; SM60-NEXT: not.b64 %rd17, %rd16;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB78_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: and.b64 %rd18, %rd7, %rd5;
+; SM60-NEXT: not.b64 %rd19, %rd18;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB79_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB79_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: and.b64 %rd20, %rd8, %rd6;
+; SM60-NEXT: not.b64 %rd21, %rd20;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB79_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB79_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: and.b64 %rd22, %rd9, %rd3;
+; SM60-NEXT: not.b64 %rd23, %rd22;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB79_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB79_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: and.b64 %rd24, %rd10, %rd4;
+; SM60-NEXT: not.b64 %rd25, %rd24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB79_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB79_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: and.b64 %rd26, %rd13, %rd1;
+; SM60-NEXT: not.b64 %rd27, %rd26;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB79_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB79_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: and.b64 %rd28, %rd14, %rd2;
+; SM60-NEXT: not.b64 %rd29, %rd28;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB79_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB79_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd30, %rd40, %rd11;
+; SM60-NEXT: not.b64 %rd31, %rd30;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB79_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB79_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b64 %rd32, %rd41, %rd12;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB79_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB82_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: or_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: or.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB83_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB85_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: or.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB86_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: or_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: or.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB87_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB87_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: or.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: or.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB87_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: or_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: or_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM60-NEXT: and.b32 %r2, %r1, 3;
+; SM60-NEXT: shl.b32 %r3, %r2, 3;
+; SM60-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r5, %r4, %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB98_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: xor.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB99_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd2, %rd1, -4;
+; SM60-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM60-NEXT: and.b32 %r3, %r2, 3;
+; SM60-NEXT: shl.b32 %r4, %r3, 3;
+; SM60-NEXT: shl.b32 %r5, %r1, %r4;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM60-NEXT: shr.u32 %r7, %r6, %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r7;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB101_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB102_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: xor.b32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB102_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB103_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: xor.b32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB103_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB103_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: xor.b32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: xor.b32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB103_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM60-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB112_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<14>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM60-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r16, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM60-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM60-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM60-NEXT: max.s16 %rs9, %rs4, %rs8;
+; SM60-NEXT: max.s16 %rs10, %rs3, %rs7;
+; SM60-NEXT: and.b16 %rs11, %rs10, 255;
+; SM60-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM60-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM60-NEXT: shl.b32 %r12, %r11, %r1;
+; SM60-NEXT: and.b32 %r13, %r16, %r2;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM60-NEXT: mov.b32 %r16, %r3;
+; SM60-NEXT: @%p1 bra $L__BB113_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM60-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM60-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM60-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM60-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM60-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM60-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM60-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p5 bra $L__BB114_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: max_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<51>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: $L__BB115_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM60-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM60-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM60-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM60-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM60-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM60-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM60-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM60-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM60-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM60-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM60-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM60-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM60-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM60-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM60-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM60-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM60-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM60-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM60-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM60-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB115_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB116_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB117_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB117_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB118_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB118_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: max_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB119_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB119_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB119_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: max.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: max.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: max.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: max.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB119_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: max_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: max_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM60-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB128_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<14>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM60-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM60-NEXT: $L__BB129_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r16, %r1;
+; SM60-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM60-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM60-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM60-NEXT: min.s16 %rs9, %rs4, %rs8;
+; SM60-NEXT: min.s16 %rs10, %rs3, %rs7;
+; SM60-NEXT: and.b16 %rs11, %rs10, 255;
+; SM60-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM60-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM60-NEXT: shl.b32 %r12, %r11, %r1;
+; SM60-NEXT: and.b32 %r13, %r16, %r2;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM60-NEXT: mov.b32 %r16, %r3;
+; SM60-NEXT: @%p1 bra $L__BB129_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM60-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM60-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM60-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM60-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM60-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM60-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM60-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p5 bra $L__BB130_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: min_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<51>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM60-NEXT: $L__BB131_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM60-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM60-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM60-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM60-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM60-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM60-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM60-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM60-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM60-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM60-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM60-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM60-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM60-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM60-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM60-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM60-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM60-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM60-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM60-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM60-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM60-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM60-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM60-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM60-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB131_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB132_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB133_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB134_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: min_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB135_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB135_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: min.s16 %rs17, %rs16, %rs14;
+; SM60-NEXT: min.s16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: min.s16 %rs23, %rs22, %rs20;
+; SM60-NEXT: min.s16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB135_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: min_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: min_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB144_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: max.u16 %rs9, %rs5, %rs7;
+; SM60-NEXT: max.u16 %rs10, %rs6, %rs8;
+; SM60-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM60-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB145_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM60-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM60-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM60-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM60-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p5 bra $L__BB146_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB147_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB148_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB149_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB150_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB150_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB151_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB151_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB151_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: max.u16 %rs17, %rs16, %rs14;
+; SM60-NEXT: max.u16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: max.u16 %rs23, %rs22, %rs20;
+; SM60-NEXT: max.u16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB151_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB160_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: min.u16 %rs9, %rs5, %rs7;
+; SM60-NEXT: min.u16 %rs10, %rs6, %rs8;
+; SM60-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM60-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB161_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM60-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM60-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM60-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM60-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p5 bra $L__BB162_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB163_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB164_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB165_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM60-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB166_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM60-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM60-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM60-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM60-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB167_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: $L__BB167_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM60-NEXT: min.u16 %rs17, %rs16, %rs14;
+; SM60-NEXT: min.u16 %rs18, %rs15, %rs13;
+; SM60-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: min.u16 %rs23, %rs22, %rs20;
+; SM60-NEXT: min.u16 %rs24, %rs21, %rs19;
+; SM60-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB167_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p2 bra $L__BB176_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM60-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: add.s16 %rs7, %rs3, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs5, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs8;
+; SM60-NEXT: selp.b16 %rs10, 0, %rs7, %p2;
+; SM60-NEXT: selp.b16 %rs11, 0, %rs6, %p1;
+; SM60-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM60-NEXT: and.b16 %rs13, %rs10, 255;
+; SM60-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB177_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<23>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, 1;
+; SM60-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, 1;
+; SM60-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, 1;
+; SM60-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM60-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM60-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM60-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM60-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM60-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM60-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM60-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM60-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM60-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM60-NEXT: mov.b32 %r22, %r1;
+; SM60-NEXT: @%p5 bra $L__BB178_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, 1;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, 1;
+; SM60-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, 1;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, 1;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM60-NEXT: add.s16 %rs10, %rs9, 1;
+; SM60-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM60-NEXT: add.s16 %rs12, %rs11, 1;
+; SM60-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM60-NEXT: add.s16 %rs14, %rs13, 1;
+; SM60-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM60-NEXT: add.s16 %rs16, %rs15, 1;
+; SM60-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM60-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM60-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM60-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM60-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM60-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM60-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM60-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB179_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.s16 %rs3, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p2 bra $L__BB180_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: $L__BB181_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs5;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs6;
+; SM60-NEXT: selp.b16 %rs7, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs8, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p3 bra $L__BB181_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB182_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM60-NEXT: add.s16 %rs7, %rs5, 1;
+; SM60-NEXT: add.s16 %rs8, %rs6, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM60-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB182_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM60-NEXT: add.s16 %rs3, %rs1, 1;
+; SM60-NEXT: add.s16 %rs4, %rs2, 1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM60-NEXT: add.s16 %rs7, %rs5, 1;
+; SM60-NEXT: add.s16 %rs8, %rs6, 1;
+; SM60-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM60-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM60-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM60-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB183_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB183_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r12;
+; SM60-NEXT: add.s16 %rs19, %rs17, 1;
+; SM60-NEXT: add.s16 %rs20, %rs18, 1;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r11;
+; SM60-NEXT: add.s16 %rs23, %rs21, 1;
+; SM60-NEXT: add.s16 %rs24, %rs22, 1;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM60-NEXT: setp.ge.u16 %p6, %rs17, %rs25;
+; SM60-NEXT: setp.ge.u16 %p7, %rs18, %rs26;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r3;
+; SM60-NEXT: setp.ge.u16 %p8, %rs21, %rs27;
+; SM60-NEXT: setp.ge.u16 %p9, %rs22, %rs28;
+; SM60-NEXT: selp.b16 %rs29, 0, %rs24, %p9;
+; SM60-NEXT: selp.b16 %rs30, 0, %rs23, %p8;
+; SM60-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs31, 0, %rs20, %p7;
+; SM60-NEXT: selp.b16 %rs32, 0, %rs19, %p6;
+; SM60-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB183_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd4, %rd6, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM60-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p2 bra $L__BB188_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd6, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM60-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p2 bra $L__BB189_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB189_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd8, %rd11, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM60-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p4 bra $L__BB189_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: add.s64 %rd10, %rd5, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM60-NEXT: @%p2 bra $L__BB190_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB190_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: add.s64 %rd12, %rd6, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM60-NEXT: @%p4 bra $L__BB190_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB190_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd14, %rd20, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM60-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p6 bra $L__BB190_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB190_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd16, %rd21, 1;
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM60-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB190_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: add.s64 %rd18, %rd7, 1;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM60-NEXT: @%p2 bra $L__BB191_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB191_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: add.s64 %rd20, %rd8, 1;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB191_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB191_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: add.s64 %rd22, %rd9, 1;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM60-NEXT: @%p6 bra $L__BB191_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB191_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: add.s64 %rd24, %rd10, 1;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM60-NEXT: @%p8 bra $L__BB191_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB191_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: add.s64 %rd26, %rd13, 1;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM60-NEXT: @%p10 bra $L__BB191_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB191_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: add.s64 %rd28, %rd14, 1;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM60-NEXT: @%p12 bra $L__BB191_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB191_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd30, %rd40, 1;
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM60-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p14 bra $L__BB191_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB191_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd32, %rd41, 1;
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM60-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p16 bra $L__BB191_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM60-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB192_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM60-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: add.s16 %rs7, %rs3, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM60-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM60-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM60-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM60-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM60-NEXT: and.b16 %rs15, %rs11, 255;
+; SM60-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p5 bra $L__BB193_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<27>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM60-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, -1;
+; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, -1;
+; SM60-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, -1;
+; SM60-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM60-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM60-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM60-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM60-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM60-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM60-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM60-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM60-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM60-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM60-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM60-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM60-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM60-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM60-NEXT: mov.b32 %r26, %r1;
+; SM60-NEXT: @%p9 bra $L__BB194_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<41>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM60-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: add.s16 %rs2, %rs1, -1;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM60-NEXT: add.s16 %rs4, %rs3, -1;
+; SM60-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM60-NEXT: add.s16 %rs6, %rs5, -1;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: add.s16 %rs8, %rs7, -1;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM60-NEXT: add.s16 %rs10, %rs9, -1;
+; SM60-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM60-NEXT: add.s16 %rs12, %rs11, -1;
+; SM60-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM60-NEXT: add.s16 %rs14, %rs13, -1;
+; SM60-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM60-NEXT: add.s16 %rs16, %rs15, -1;
+; SM60-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM60-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM60-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM60-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM60-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM60-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM60-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM60-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM60-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM60-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM60-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM60-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM60-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM60-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM60-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM60-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM60-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM60-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM60-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM60-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM60-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM60-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM60-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM60-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB195_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.s16 %rs3, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM60-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p3 bra $L__BB196_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<11>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM60-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM60-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM60-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p5 bra $L__BB197_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<21>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB198_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM60-NEXT: add.s16 %rs7, %rs5, -1;
+; SM60-NEXT: add.s16 %rs8, %rs6, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM60-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM60-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM60-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM60-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM60-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM60-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM60-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB198_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b16 %rs<41>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM60-NEXT: $L__BB199_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM60-NEXT: add.s16 %rs3, %rs1, -1;
+; SM60-NEXT: add.s16 %rs4, %rs2, -1;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM60-NEXT: add.s16 %rs7, %rs5, -1;
+; SM60-NEXT: add.s16 %rs8, %rs6, -1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM60-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM60-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM60-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM60-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM60-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM60-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM60-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM60-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs16, %rs14};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM60-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM60-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM60-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs20, %rs18};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB199_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB199_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM60-NEXT: add.s16 %rs23, %rs21, -1;
+; SM60-NEXT: add.s16 %rs24, %rs22, -1;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r11;
+; SM60-NEXT: add.s16 %rs27, %rs25, -1;
+; SM60-NEXT: add.s16 %rs28, %rs26, -1;
+; SM60-NEXT: setp.eq.b16 %p10, %rs21, 0;
+; SM60-NEXT: setp.eq.b16 %p11, %rs22, 0;
+; SM60-NEXT: setp.eq.b16 %p12, %rs25, 0;
+; SM60-NEXT: setp.eq.b16 %p13, %rs26, 0;
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r4;
+; SM60-NEXT: setp.gt.u16 %p14, %rs21, %rs29;
+; SM60-NEXT: setp.gt.u16 %p15, %rs22, %rs30;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r3;
+; SM60-NEXT: setp.gt.u16 %p16, %rs25, %rs31;
+; SM60-NEXT: setp.gt.u16 %p17, %rs26, %rs32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs28, %p17;
+; SM60-NEXT: selp.b16 %rs34, %rs32, %rs33, %p13;
+; SM60-NEXT: selp.b16 %rs35, %rs31, %rs27, %p16;
+; SM60-NEXT: selp.b16 %rs36, %rs31, %rs35, %p12;
+; SM60-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs37, %rs30, %rs24, %p15;
+; SM60-NEXT: selp.b16 %rs38, %rs30, %rs37, %p11;
+; SM60-NEXT: selp.b16 %rs39, %rs29, %rs23, %p14;
+; SM60-NEXT: selp.b16 %rs40, %rs29, %rs39, %p10;
+; SM60-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB199_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b64 %rd<8>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd7, [%rd3];
+; SM60-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd4, %rd7, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM60-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM60-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM60-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM60-NEXT: mov.b64 %rd7, %rd2;
+; SM60-NEXT: @%p3 bra $L__BB204_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b64 %rd<14>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd5];
+; SM60-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd6, %rd12, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM60-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM60-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM60-NEXT: mov.b64 %rd12, %rd3;
+; SM60-NEXT: @%p3 bra $L__BB205_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd5+8];
+; SM60-NEXT: $L__BB205_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd9, %rd13, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd13, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM60-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM60-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
+; SM60-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
+; SM60-NEXT: mov.b64 %rd13, %rd4;
+; SM60-NEXT: @%p6 bra $L__BB205_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<13>;
+; SM60-NEXT: .reg .b64 %rd<26>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd9];
+; SM60-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd22;
+; SM60-NEXT: add.s64 %rd10, %rd5, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
+; SM60-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
+; SM60-NEXT: @%p3 bra $L__BB206_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd23, [%rd9+8];
+; SM60-NEXT: $L__BB206_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd23;
+; SM60-NEXT: add.s64 %rd13, %rd6, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM60-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
+; SM60-NEXT: @%p6 bra $L__BB206_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd9+16];
+; SM60-NEXT: $L__BB206_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd16, %rd24, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
+; SM60-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
+; SM60-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM60-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM60-NEXT: mov.b64 %rd24, %rd7;
+; SM60-NEXT: @%p9 bra $L__BB206_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd25, [%rd9+24];
+; SM60-NEXT: $L__BB206_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd19, %rd25, -1;
+; SM60-NEXT: setp.eq.b64 %p10, %rd25, 0;
+; SM60-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM60-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM60-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
+; SM60-NEXT: mov.b64 %rd25, %rd8;
+; SM60-NEXT: @%p12 bra $L__BB206_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<25>;
+; SM60-NEXT: .reg .b64 %rd<50>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd42, [%rd17];
+; SM60-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd42;
+; SM60-NEXT: add.s64 %rd18, %rd7, -1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM60-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB207_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd43, [%rd17+8];
+; SM60-NEXT: $L__BB207_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd43;
+; SM60-NEXT: add.s64 %rd21, %rd8, -1;
+; SM60-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM60-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM60-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
+; SM60-NEXT: @%p6 bra $L__BB207_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd44, [%rd17+16];
+; SM60-NEXT: $L__BB207_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd44;
+; SM60-NEXT: add.s64 %rd24, %rd9, -1;
+; SM60-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM60-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM60-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM60-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
+; SM60-NEXT: @%p9 bra $L__BB207_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd45, [%rd17+24];
+; SM60-NEXT: $L__BB207_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd45;
+; SM60-NEXT: add.s64 %rd27, %rd10, -1;
+; SM60-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM60-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM60-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
+; SM60-NEXT: @%p12 bra $L__BB207_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd46, [%rd17+32];
+; SM60-NEXT: $L__BB207_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd46;
+; SM60-NEXT: add.s64 %rd30, %rd13, -1;
+; SM60-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM60-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
+; SM60-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
+; SM60-NEXT: @%p15 bra $L__BB207_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd47, [%rd17+40];
+; SM60-NEXT: $L__BB207_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd47;
+; SM60-NEXT: add.s64 %rd33, %rd14, -1;
+; SM60-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM60-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM60-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM60-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
+; SM60-NEXT: @%p18 bra $L__BB207_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd48, [%rd17+48];
+; SM60-NEXT: $L__BB207_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd36, %rd48, -1;
+; SM60-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM60-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM60-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM60-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM60-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM60-NEXT: mov.b64 %rd48, %rd15;
+; SM60-NEXT: @%p21 bra $L__BB207_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd49, [%rd17+56];
+; SM60-NEXT: $L__BB207_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s64 %rd39, %rd49, -1;
+; SM60-NEXT: setp.eq.b64 %p22, %rd49, 0;
+; SM60-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM60-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM60-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
+; SM60-NEXT: mov.b64 %rd49, %rd16;
+; SM60-NEXT: @%p24 bra $L__BB207_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: and.b32 %r10, %r9, 255;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p2 bra $L__BB208_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM60-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: and.b16 %rs4, %rs3, 255;
+; SM60-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM60-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM60-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM60-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM60-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM60-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM60-NEXT: and.b16 %rs13, %rs10, 255;
+; SM60-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p3 bra $L__BB209_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<23>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM60-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM60-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM60-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM60-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM60-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM60-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM60-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM60-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM60-NEXT: mov.b32 %r22, %r1;
+; SM60-NEXT: @%p5 bra $L__BB210_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM60-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM60-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM60-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM60-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM60-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM60-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM60-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM60-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM60-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM60-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM60-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM60-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM60-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM60-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM60-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM60-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM60-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM60-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM60-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM60-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM60-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM60-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM60-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM60-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB211_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p2 bra $L__BB212_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p3 bra $L__BB213_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM60-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM60-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM60-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM60-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM60-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB214_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM60-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM60-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r9;
+; SM60-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM60-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM60-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM60-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM60-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM60-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM60-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM60-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM60-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB215_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB215_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r12;
+; SM60-NEXT: setp.ge.u16 %p6, %rs19, %rs17;
+; SM60-NEXT: setp.ge.u16 %p7, %rs20, %rs18;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r3;
+; SM60-NEXT: mov.b32 {%rs23, %rs24}, %r11;
+; SM60-NEXT: setp.ge.u16 %p8, %rs23, %rs21;
+; SM60-NEXT: setp.ge.u16 %p9, %rs24, %rs22;
+; SM60-NEXT: sub.s16 %rs25, %rs19, %rs17;
+; SM60-NEXT: sub.s16 %rs26, %rs20, %rs18;
+; SM60-NEXT: sub.s16 %rs27, %rs23, %rs21;
+; SM60-NEXT: sub.s16 %rs28, %rs24, %rs22;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs24, %p9;
+; SM60-NEXT: selp.b16 %rs30, %rs27, %rs23, %p8;
+; SM60-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM60-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM60-NEXT: selp.b16 %rs31, %rs26, %rs20, %p7;
+; SM60-NEXT: selp.b16 %rs32, %rs25, %rs19, %p6;
+; SM60-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB215_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM60-NEXT: sub.s32 %r3, %r5, %r1;
+; SM60-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p2 bra $L__BB216_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB217_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM60-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM60-NEXT: sub.s32 %r3, %r8, %r2;
+; SM60-NEXT: sub.s32 %r4, %r7, %r1;
+; SM60-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB217_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB218_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r14, %r2;
+; SM60-NEXT: setp.ge.u32 %p2, %r13, %r1;
+; SM60-NEXT: sub.s32 %r5, %r14, %r2;
+; SM60-NEXT: sub.s32 %r6, %r13, %r1;
+; SM60-NEXT: selp.b32 %r7, %r6, %r13, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: selp.b32 %r8, %r5, %r14, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB218_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB218_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: setp.ge.u32 %p4, %r16, %r4;
+; SM60-NEXT: setp.ge.u32 %p5, %r15, %r3;
+; SM60-NEXT: sub.s32 %r9, %r16, %r4;
+; SM60-NEXT: sub.s32 %r10, %r15, %r3;
+; SM60-NEXT: selp.b32 %r11, %r10, %r15, %p5;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: selp.b32 %r12, %r9, %r16, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p6 bra $L__BB218_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<13>;
+; SM60-NEXT: .reg .b32 %r<33>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r26;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: setp.ge.u32 %p1, %r26, %r6;
+; SM60-NEXT: setp.ge.u32 %p2, %r25, %r5;
+; SM60-NEXT: sub.s32 %r9, %r26, %r6;
+; SM60-NEXT: sub.s32 %r10, %r25, %r5;
+; SM60-NEXT: selp.b32 %r11, %r10, %r25, %p2;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: selp.b32 %r12, %r9, %r26, %p1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p3 bra $L__BB219_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: $L__BB219_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd13, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: setp.ge.u32 %p4, %r28, %r8;
+; SM60-NEXT: setp.ge.u32 %p5, %r27, %r7;
+; SM60-NEXT: sub.s32 %r13, %r28, %r8;
+; SM60-NEXT: sub.s32 %r14, %r27, %r7;
+; SM60-NEXT: selp.b32 %r15, %r14, %r27, %p5;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NEXT: selp.b32 %r16, %r13, %r28, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM60-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p6 bra $L__BB219_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM60-NEXT: $L__BB219_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd23, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r30;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: setp.ge.u32 %p7, %r30, %r2;
+; SM60-NEXT: setp.ge.u32 %p8, %r29, %r1;
+; SM60-NEXT: sub.s32 %r17, %r30, %r2;
+; SM60-NEXT: sub.s32 %r18, %r29, %r1;
+; SM60-NEXT: selp.b32 %r19, %r18, %r29, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r19;
+; SM60-NEXT: selp.b32 %r20, %r17, %r30, %p7;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r20;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd26, %rd30;
+; SM60-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM60-NEXT: @%p9 bra $L__BB219_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM60-NEXT: $L__BB219_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: setp.ge.u32 %p10, %r32, %r4;
+; SM60-NEXT: setp.ge.u32 %p11, %r31, %r3;
+; SM60-NEXT: sub.s32 %r21, %r32, %r4;
+; SM60-NEXT: sub.s32 %r22, %r31, %r3;
+; SM60-NEXT: selp.b32 %r23, %r22, %r31, %p11;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: selp.b32 %r24, %r21, %r32, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd36, %rd40;
+; SM60-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM60-NEXT: @%p12 bra $L__BB219_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM60-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM60-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p2 bra $L__BB220_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM60-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p2 bra $L__BB221_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd8, %rd11, %rd2;
+; SM60-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p4 bra $L__BB221_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM60-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM60-NEXT: @%p2 bra $L__BB222_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB222_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM60-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM60-NEXT: @%p4 bra $L__BB222_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB222_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM60-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM60-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p6 bra $L__BB222_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB222_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd16, %rd21, %rd2;
+; SM60-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB222_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM60-NEXT: @%p2 bra $L__BB223_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB223_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM60-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB223_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB223_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM60-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM60-NEXT: @%p6 bra $L__BB223_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB223_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM60-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM60-NEXT: @%p8 bra $L__BB223_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB223_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM60-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM60-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM60-NEXT: @%p10 bra $L__BB223_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB223_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM60-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM60-NEXT: @%p12 bra $L__BB223_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB223_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM60-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p14 bra $L__BB223_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB223_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd32, %rd41, %rd12;
+; SM60-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p16 bra $L__BB223_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<6>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 255;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: and.b16 %rs3, %rs2, 255;
+; SM60-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB224_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<22>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r21, [%rd1];
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM60-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r21, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM60-NEXT: sub.s16 %rs9, %rs6, %rs8;
+; SM60-NEXT: setp.gt.u16 %p1, %rs9, %rs6;
+; SM60-NEXT: selp.b16 %rs10, -1, 0, %p1;
+; SM60-NEXT: sub.s16 %rs11, %rs5, %rs7;
+; SM60-NEXT: setp.gt.u16 %p2, %rs11, %rs5;
+; SM60-NEXT: selp.b16 %rs12, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r12, {%rs12, %rs10};
+; SM60-NEXT: xor.b32 %r13, %r12, -1;
+; SM60-NEXT: mov.b32 %r14, {%rs11, %rs9};
+; SM60-NEXT: and.b32 %r15, %r14, %r13;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r15;
+; SM60-NEXT: shl.b16 %rs15, %rs14, 8;
+; SM60-NEXT: or.b16 %rs16, %rs13, %rs15;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, %r1;
+; SM60-NEXT: and.b32 %r18, %r21, %r2;
+; SM60-NEXT: or.b32 %r19, %r18, %r17;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
+; SM60-NEXT: setp.ne.b32 %p3, %r3, %r21;
+; SM60-NEXT: mov.b32 %r21, %r3;
+; SM60-NEXT: @%p3 bra $L__BB225_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r20, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r20;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM60-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM60-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM60-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB226_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM60-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM60-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM60-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM60-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM60-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM60-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM60-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM60-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM60-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM60-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM60-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM60-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM60-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM60-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB227_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB228_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<8>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r7;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: and.b32 %r6, %r3, %r5;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r7;
+; SM60-NEXT: mov.b32 %r7, %r1;
+; SM60-NEXT: @%p3 bra $L__BB229_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r11;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: and.b32 %r6, %r3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r12;
+; SM60-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM60-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM60-NEXT: mov.b32 %r7, {%rs14, %rs13};
+; SM60-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM60-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM60-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM60-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM60-NEXT: mov.b32 %r8, {%rs16, %rs15};
+; SM60-NEXT: xor.b32 %r9, %r8, -1;
+; SM60-NEXT: and.b32 %r10, %r7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB230_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<33>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM60-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM60-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM60-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM60-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM60-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM60-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM60-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM60-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM60-NEXT: mov.b32 %r6, {%rs8, %rs7};
+; SM60-NEXT: xor.b32 %r7, %r6, -1;
+; SM60-NEXT: and.b32 %r8, %r5, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r22;
+; SM60-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM60-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM60-NEXT: mov.b32 %r9, {%rs14, %rs13};
+; SM60-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM60-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM60-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM60-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM60-NEXT: mov.b32 %r10, {%rs16, %rs15};
+; SM60-NEXT: xor.b32 %r11, %r10, -1;
+; SM60-NEXT: and.b32 %r12, %r9, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB231_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd12;
+; SM60-NEXT: $L__BB231_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r3;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r23;
+; SM60-NEXT: sub.s16 %rs21, %rs20, %rs18;
+; SM60-NEXT: sub.s16 %rs22, %rs19, %rs17;
+; SM60-NEXT: mov.b32 %r13, {%rs22, %rs21};
+; SM60-NEXT: setp.gt.u16 %p6, %rs21, %rs20;
+; SM60-NEXT: selp.b16 %rs23, -1, 0, %p6;
+; SM60-NEXT: setp.gt.u16 %p7, %rs22, %rs19;
+; SM60-NEXT: selp.b16 %rs24, -1, 0, %p7;
+; SM60-NEXT: mov.b32 %r14, {%rs24, %rs23};
+; SM60-NEXT: xor.b32 %r15, %r14, -1;
+; SM60-NEXT: and.b32 %r16, %r13, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r24;
+; SM60-NEXT: sub.s16 %rs29, %rs28, %rs26;
+; SM60-NEXT: sub.s16 %rs30, %rs27, %rs25;
+; SM60-NEXT: mov.b32 %r17, {%rs30, %rs29};
+; SM60-NEXT: setp.gt.u16 %p8, %rs29, %rs28;
+; SM60-NEXT: selp.b16 %rs31, -1, 0, %p8;
+; SM60-NEXT: setp.gt.u16 %p9, %rs30, %rs27;
+; SM60-NEXT: selp.b16 %rs32, -1, 0, %p9;
+; SM60-NEXT: mov.b32 %r18, {%rs32, %rs31};
+; SM60-NEXT: xor.b32 %r19, %r18, -1;
+; SM60-NEXT: and.b32 %r20, %r17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r20;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r24;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB231_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r3, %r5, %r1;
+; SM60-NEXT: sub.s32 %r4, %r3, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB232_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r3, %r7, %r1;
+; SM60-NEXT: sub.s32 %r4, %r3, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM60-NEXT: max.u32 %r5, %r8, %r2;
+; SM60-NEXT: sub.s32 %r6, %r5, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB233_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r5, %r13, %r1;
+; SM60-NEXT: sub.s32 %r6, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: max.u32 %r7, %r14, %r2;
+; SM60-NEXT: sub.s32 %r8, %r7, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB234_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM60-NEXT: $L__BB234_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r9, %r15, %r3;
+; SM60-NEXT: sub.s32 %r10, %r9, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM60-NEXT: max.u32 %r11, %r16, %r4;
+; SM60-NEXT: sub.s32 %r12, %r11, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB234_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<33>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r9, %r25, %r5;
+; SM60-NEXT: sub.s32 %r10, %r9, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r10;
+; SM60-NEXT: max.u32 %r11, %r26, %r6;
+; SM60-NEXT: sub.s32 %r12, %r11, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB235_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: $L__BB235_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r13, %r27, %r7;
+; SM60-NEXT: sub.s32 %r14, %r13, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r14;
+; SM60-NEXT: max.u32 %r15, %r28, %r8;
+; SM60-NEXT: sub.s32 %r16, %r15, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB235_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM60-NEXT: $L__BB235_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r17, %r29, %r1;
+; SM60-NEXT: sub.s32 %r18, %r17, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r18;
+; SM60-NEXT: max.u32 %r19, %r30, %r2;
+; SM60-NEXT: sub.s32 %r20, %r19, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r20;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r30;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB235_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM60-NEXT: $L__BB235_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u32 %r21, %r31, %r3;
+; SM60-NEXT: sub.s32 %r22, %r21, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM60-NEXT: max.u32 %r23, %r32, %r4;
+; SM60-NEXT: sub.s32 %r24, %r23, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r32;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB235_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<7>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd6, [%rd3];
+; SM60-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM60-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM60-NEXT: mov.b64 %rd6, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB236_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd10, [%rd5];
+; SM60-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM60-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM60-NEXT: mov.b64 %rd10, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB237_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd11, [%rd5+8];
+; SM60-NEXT: $L__BB237_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM60-NEXT: sub.s64 %rd9, %rd8, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM60-NEXT: mov.b64 %rd11, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB237_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd18, [%rd9];
+; SM60-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd18;
+; SM60-NEXT: max.u64 %rd10, %rd5, %rd3;
+; SM60-NEXT: sub.s64 %rd11, %rd10, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB238_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd19, [%rd9+8];
+; SM60-NEXT: $L__BB238_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd19;
+; SM60-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM60-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM60-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB238_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd20, [%rd9+16];
+; SM60-NEXT: $L__BB238_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd14, %rd20, %rd1;
+; SM60-NEXT: sub.s64 %rd15, %rd14, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM60-NEXT: mov.b64 %rd20, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB238_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd21, [%rd9+24];
+; SM60-NEXT: $L__BB238_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM60-NEXT: sub.s64 %rd17, %rd16, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM60-NEXT: mov.b64 %rd21, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB238_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM60-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd34, [%rd17];
+; SM60-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd34;
+; SM60-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM60-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM60-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB239_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd17+8];
+; SM60-NEXT: $L__BB239_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd35;
+; SM60-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM60-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM60-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB239_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd36, [%rd17+16];
+; SM60-NEXT: $L__BB239_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd36;
+; SM60-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM60-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM60-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB239_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd37, [%rd17+24];
+; SM60-NEXT: $L__BB239_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd37;
+; SM60-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM60-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM60-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB239_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd38, [%rd17+32];
+; SM60-NEXT: $L__BB239_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd38;
+; SM60-NEXT: max.u64 %rd26, %rd13, %rd1;
+; SM60-NEXT: sub.s64 %rd27, %rd26, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM60-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB239_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd39, [%rd17+40];
+; SM60-NEXT: $L__BB239_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd39;
+; SM60-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM60-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM60-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB239_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd40, [%rd17+48];
+; SM60-NEXT: $L__BB239_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM60-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM60-NEXT: mov.b64 %rd40, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB239_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd41, [%rd17+56];
+; SM60-NEXT: $L__BB239_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM60-NEXT: sub.s64 %rd33, %rd32, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM60-NEXT: mov.b64 %rd41, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB239_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB244_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB245_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB246_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB246_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB246_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: sub.rn.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB247_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB247_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: sub.rn.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB247_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB247_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: sub.rn.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB247_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB247_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: sub.rn.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB247_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB248_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: min.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB249_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: min.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB250_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB250_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: min.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB250_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: min.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB251_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB251_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: min.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB251_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB251_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: min.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB251_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB251_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: min.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB251_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r3, %r4, %r1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM60-NEXT: mov.b32 %r4, %r2;
+; SM60-NEXT: @%p1 bra $L__BB252_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: max.f32 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB253_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: max.f32 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB254_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB254_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: max.f32 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB254_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r9, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: max.f32 %r10, %r18, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB255_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM60-NEXT: $L__BB255_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r11, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM60-NEXT: max.f32 %r12, %r20, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB255_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM60-NEXT: $L__BB255_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r13, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM60-NEXT: max.f32 %r14, %r22, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM60-NEXT: @%p3 bra $L__BB255_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM60-NEXT: $L__BB255_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f32 %r15, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM60-NEXT: max.f32 %r16, %r24, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM60-NEXT: @%p4 bra $L__BB255_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM60-NEXT: min.f32 %r3, %r8, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM60-NEXT: mov.b32 %r8, %r2;
+; SM60-NEXT: @%p5 bra $L__BB256_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM60-NEXT: min.f32 %r3, %r13, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
+; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM60-NEXT: min.f32 %r8, %r14, %r2;
+; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
+; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB257_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM60-NEXT: min.f32 %r5, %r25, %r1;
+; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
+; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM60-NEXT: min.f32 %r10, %r26, %r2;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
+; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB258_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM60-NEXT: $L__BB258_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM60-NEXT: min.f32 %r15, %r27, %r3;
+; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
+; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM60-NEXT: min.f32 %r20, %r28, %r4;
+; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
+; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB258_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<37>;
+; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM60-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM60-NEXT: min.f32 %r9, %r49, %r5;
+; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
+; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM60-NEXT: min.f32 %r14, %r50, %r6;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
+; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB259_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM60-NEXT: $L__BB259_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM60-NEXT: min.f32 %r19, %r51, %r7;
+; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
+; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM60-NEXT: min.f32 %r24, %r52, %r8;
+; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
+; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB259_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM60-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM60-NEXT: $L__BB259_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM60-NEXT: min.f32 %r29, %r53, %r1;
+; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM60-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
+; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM60-NEXT: min.f32 %r34, %r54, %r2;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
+; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM60-NEXT: @%p27 bra $L__BB259_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM60-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
+; SM60-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM60-NEXT: $L__BB259_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM60-NEXT: min.f32 %r39, %r55, %r3;
+; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM60-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
+; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM60-NEXT: min.f32 %r44, %r56, %r4;
+; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM60-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
+; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM60-NEXT: @%p36 bra $L__BB259_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM60-NEXT: max.f32 %r3, %r8, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM60-NEXT: mov.b32 %r8, %r2;
+; SM60-NEXT: @%p5 bra $L__BB260_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM60-NEXT: max.f32 %r3, %r13, %r1;
+; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r13, 0;
+; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM60-NEXT: max.f32 %r8, %r14, %r2;
+; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r14, 0;
+; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB261_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<19>;
+; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM60-NEXT: max.f32 %r5, %r25, %r1;
+; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r25, 0;
+; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM60-NEXT: max.f32 %r10, %r26, %r2;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r26, 0;
+; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB262_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM60-NEXT: $L__BB262_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM60-NEXT: max.f32 %r15, %r27, %r3;
+; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r27, 0;
+; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM60-NEXT: max.f32 %r20, %r28, %r4;
+; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r28, 0;
+; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB262_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<37>;
+; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .b64 %rd<42>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM60-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM60-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM60-NEXT: max.f32 %r9, %r49, %r5;
+; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM60-NEXT: setp.eq.b32 %p2, %r49, 0;
+; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM60-NEXT: max.f32 %r14, %r50, %r6;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM60-NEXT: setp.eq.b32 %p6, %r50, 0;
+; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM60-NEXT: @%p9 bra $L__BB263_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM60-NEXT: setp.eq.b32 %p12, %r7, 0;
+; SM60-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM60-NEXT: $L__BB263_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM60-NEXT: max.f32 %r19, %r51, %r7;
+; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM60-NEXT: setp.eq.b32 %p11, %r51, 0;
+; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM60-NEXT: max.f32 %r24, %r52, %r8;
+; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM60-NEXT: setp.eq.b32 %p15, %r52, 0;
+; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM60-NEXT: @%p18 bra $L__BB263_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM60-NEXT: setp.eq.b32 %p21, %r1, 0;
+; SM60-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM60-NEXT: $L__BB263_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM60-NEXT: max.f32 %r29, %r53, %r1;
+; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM60-NEXT: setp.eq.b32 %p20, %r53, 0;
+; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM60-NEXT: max.f32 %r34, %r54, %r2;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.b32 %p24, %r54, 0;
+; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM60-NEXT: @%p27 bra $L__BB263_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM60-NEXT: setp.eq.b32 %p30, %r3, 0;
+; SM60-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM60-NEXT: $L__BB263_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM60-NEXT: max.f32 %r39, %r55, %r3;
+; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM60-NEXT: setp.eq.b32 %p29, %r55, 0;
+; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM60-NEXT: max.f32 %r44, %r56, %r4;
+; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM60-NEXT: setp.eq.b32 %p33, %r56, 0;
+; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM60-NEXT: @%p36 bra $L__BB263_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<4>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM60-NEXT: atom.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM60-NEXT: atom.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM60-NEXT: atom.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM60-NEXT: atom.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM60-NEXT: atom.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM60-NEXT: atom.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM60-NEXT: atom.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM60-NEXT: atom.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB268_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB269_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB269_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB269_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB269_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB270_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB270_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB270_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB270_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB270_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB270_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB270_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB270_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB271_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB271_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB271_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB271_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB271_5: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB271_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end19
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB271_7: // %atomicrmw.start26
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB271_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end25
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB271_9: // %atomicrmw.start41
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB271_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end40
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB271_11: // %atomicrmw.start47
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB271_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end46
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB271_13: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB271_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end57
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB271_15: // %atomicrmw.start64
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB271_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end63
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB272_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB273_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB273_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB273_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB273_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB274_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB274_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB274_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB274_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB274_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB274_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB274_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB274_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB275_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB275_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB275_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB275_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB275_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB275_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB275_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB275_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB275_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB275_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB275_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB275_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB275_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB275_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB275_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: min.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB275_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b64 %rd<6>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd5, [%rd3];
+; SM60-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM60-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM60-NEXT: mov.b64 %rd5, %rd2;
+; SM60-NEXT: @%p1 bra $L__BB276_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd5];
+; SM60-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd3;
+; SM60-NEXT: @%p1 bra $L__BB277_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd5+8];
+; SM60-NEXT: $L__BB277_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd7, %rd9, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM60-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd4;
+; SM60-NEXT: @%p2 bra $L__BB277_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd9];
+; SM60-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd14;
+; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM60-NEXT: @%p1 bra $L__BB278_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd9+8];
+; SM60-NEXT: $L__BB278_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd15;
+; SM60-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM60-NEXT: @%p2 bra $L__BB278_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd9+16];
+; SM60-NEXT: $L__BB278_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM60-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd7;
+; SM60-NEXT: @%p3 bra $L__BB278_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd9+24];
+; SM60-NEXT: $L__BB278_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd13, %rd17, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd8;
+; SM60-NEXT: @%p4 bra $L__BB278_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd17];
+; SM60-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd26;
+; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM60-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM60-NEXT: @%p1 bra $L__BB279_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd17+8];
+; SM60-NEXT: $L__BB279_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd27;
+; SM60-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM60-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM60-NEXT: @%p2 bra $L__BB279_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd17+16];
+; SM60-NEXT: $L__BB279_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd28;
+; SM60-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM60-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM60-NEXT: @%p3 bra $L__BB279_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd17+24];
+; SM60-NEXT: $L__BB279_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd29;
+; SM60-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM60-NEXT: @%p4 bra $L__BB279_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd17+32];
+; SM60-NEXT: $L__BB279_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd30;
+; SM60-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM60-NEXT: @%p5 bra $L__BB279_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd17+40];
+; SM60-NEXT: $L__BB279_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd31;
+; SM60-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM60-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM60-NEXT: @%p6 bra $L__BB279_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd17+48];
+; SM60-NEXT: $L__BB279_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd15;
+; SM60-NEXT: @%p7 bra $L__BB279_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd17+56];
+; SM60-NEXT: $L__BB279_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: max.f64 %rd25, %rd33, %rd12;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd16;
+; SM60-NEXT: @%p8 bra $L__BB279_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM60-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd2;
+; SM60-NEXT: @%p5 bra $L__BB280_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM60-NEXT: min.f64 %rd6, %rd16, %rd1;
+; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd3;
+; SM60-NEXT: @%p5 bra $L__BB281_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB281_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM60-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd4;
+; SM60-NEXT: @%p10 bra $L__BB281_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<21>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
+; SM60-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd30;
+; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM60-NEXT: @%p5 bra $L__BB282_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM60-NEXT: $L__BB282_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: min.f64 %rd15, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM60-NEXT: @%p10 bra $L__BB282_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB282_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM60-NEXT: min.f64 %rd20, %rd32, %rd1;
+; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd7;
+; SM60-NEXT: @%p15 bra $L__BB282_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB282_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM60-NEXT: min.f64 %rd25, %rd33, %rd2;
+; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd8;
+; SM60-NEXT: @%p20 bra $L__BB282_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<41>;
+; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
+; SM60-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd58;
+; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM60-NEXT: @%p5 bra $L__BB283_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM60-NEXT: $L__BB283_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: min.f64 %rd23, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM60-NEXT: @%p10 bra $L__BB283_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM60-NEXT: $L__BB283_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd60;
+; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: min.f64 %rd28, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM60-NEXT: @%p15 bra $L__BB283_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM60-NEXT: $L__BB283_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd61;
+; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM60-NEXT: min.f64 %rd33, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM60-NEXT: @%p20 bra $L__BB283_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
+; SM60-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM60-NEXT: $L__BB283_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd62;
+; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM60-NEXT: min.f64 %rd38, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM60-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM60-NEXT: @%p25 bra $L__BB283_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
+; SM60-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM60-NEXT: $L__BB283_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd63;
+; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM60-NEXT: min.f64 %rd43, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM60-NEXT: @%p30 bra $L__BB283_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
+; SM60-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM60-NEXT: $L__BB283_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM60-NEXT: min.f64 %rd48, %rd64, %rd11;
+; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM60-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM60-NEXT: mov.b64 %rd64, %rd15;
+; SM60-NEXT: @%p35 bra $L__BB283_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
+; SM60-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM60-NEXT: $L__BB283_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM60-NEXT: min.f64 %rd53, %rd65, %rd12;
+; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM60-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM60-NEXT: mov.b64 %rd65, %rd16;
+; SM60-NEXT: @%p40 bra $L__BB283_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM60-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM60-NEXT: mov.b64 %rd9, %rd2;
+; SM60-NEXT: @%p5 bra $L__BB284_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
+; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM60-NEXT: max.f64 %rd6, %rd16, %rd1;
+; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM60-NEXT: mov.b64 %rd16, %rd3;
+; SM60-NEXT: @%p5 bra $L__BB285_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM60-NEXT: $L__BB285_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM60-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd17, 0;
+; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM60-NEXT: mov.b64 %rd17, %rd4;
+; SM60-NEXT: @%p10 bra $L__BB285_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<21>;
+; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
+; SM60-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd5, %rd30;
+; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd5, 0;
+; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM60-NEXT: @%p5 bra $L__BB286_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM60-NEXT: $L__BB286_3: // %atomicrmw.start5
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM60-NEXT: max.f64 %rd15, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM60-NEXT: @%p10 bra $L__BB286_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end4
+; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM60-NEXT: $L__BB286_5: // %atomicrmw.start15
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM60-NEXT: max.f64 %rd20, %rd32, %rd1;
+; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd32, 0;
+; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM60-NEXT: mov.b64 %rd32, %rd7;
+; SM60-NEXT: @%p15 bra $L__BB286_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end14
+; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM60-NEXT: $L__BB286_7: // %atomicrmw.start20
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM60-NEXT: max.f64 %rd25, %rd33, %rd2;
+; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd33, 0;
+; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM60-NEXT: mov.b64 %rd33, %rd8;
+; SM60-NEXT: @%p20 bra $L__BB286_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end19
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<41>;
+; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
+; SM60-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd7, %rd58;
+; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM60-NEXT: setp.eq.b64 %p2, %rd7, 0;
+; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM60-NEXT: @%p5 bra $L__BB287_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
+; SM60-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM60-NEXT: $L__BB287_3: // %atomicrmw.start9
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM60-NEXT: max.f64 %rd23, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM60-NEXT: setp.eq.b64 %p7, %rd8, 0;
+; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM60-NEXT: @%p10 bra $L__BB287_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end8
+; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
+; SM60-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM60-NEXT: $L__BB287_5: // %atomicrmw.start19
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd9, %rd60;
+; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: max.f64 %rd28, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM60-NEXT: setp.eq.b64 %p12, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM60-NEXT: @%p15 bra $L__BB287_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end18
+; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
+; SM60-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM60-NEXT: $L__BB287_7: // %atomicrmw.start24
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd10, %rd61;
+; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM60-NEXT: max.f64 %rd33, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM60-NEXT: setp.eq.b64 %p17, %rd10, 0;
+; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM60-NEXT: @%p20 bra $L__BB287_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end23
+; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
+; SM60-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM60-NEXT: $L__BB287_9: // %atomicrmw.start38
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd13, %rd62;
+; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM60-NEXT: max.f64 %rd38, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM60-NEXT: setp.eq.b64 %p22, %rd13, 0;
+; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM60-NEXT: @%p25 bra $L__BB287_9;
+; SM60-NEXT: // %bb.10: // %atomicrmw.end37
+; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
+; SM60-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM60-NEXT: $L__BB287_11: // %atomicrmw.start43
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b64 %rd14, %rd63;
+; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM60-NEXT: max.f64 %rd43, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.b64 %p27, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM60-NEXT: @%p30 bra $L__BB287_11;
+; SM60-NEXT: // %bb.12: // %atomicrmw.end42
+; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
+; SM60-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM60-NEXT: $L__BB287_13: // %atomicrmw.start53
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM60-NEXT: max.f64 %rd48, %rd64, %rd11;
+; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM60-NEXT: setp.eq.b64 %p32, %rd64, 0;
+; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM60-NEXT: mov.b64 %rd64, %rd15;
+; SM60-NEXT: @%p35 bra $L__BB287_13;
+; SM60-NEXT: // %bb.14: // %atomicrmw.end52
+; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
+; SM60-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM60-NEXT: $L__BB287_15: // %atomicrmw.start58
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM60-NEXT: max.f64 %rd53, %rd65, %rd12;
+; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM60-NEXT: setp.eq.b64 %p37, %rd65, 0;
+; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM60-NEXT: mov.b64 %rd65, %rd16;
+; SM60-NEXT: @%p40 bra $L__BB287_15;
+; SM60-NEXT: // %bb.16: // %atomicrmw.end57
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB288_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB288_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB289_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB289_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB290_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: add.rn.f16x2 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB290_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB291_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: add.rn.f16x2 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB291_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB291_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.rn.f16x2 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: add.rn.f16x2 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB291_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p1 bra $L__BB292_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p1 bra $L__BB293_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB294_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB295_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB295_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NEXT: min.f32 %r11, %r9, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB296_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<11>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r10, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: min.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: min.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM60-NEXT: mov.b32 %r10, %r1;
+; SM60-NEXT: @%p1 bra $L__BB297_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM60-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: min.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: min.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM60-NEXT: min.f32 %r12, %r11, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM60-NEXT: min.f32 %r15, %r14, %r13;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM60-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB298_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<37>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM60-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM60-NEXT: min.f32 %r7, %r6, %r5;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM60-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM60-NEXT: min.f32 %r10, %r9, %r8;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM60-NEXT: min.f32 %r14, %r13, %r12;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM60-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM60-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM60-NEXT: min.f32 %r17, %r16, %r15;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM60-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB299_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM60-NEXT: $L__BB299_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM60-NEXT: min.f32 %r21, %r20, %r19;
+; SM60-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM60-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM60-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM60-NEXT: min.f32 %r24, %r23, %r22;
+; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM60-NEXT: min.f32 %r28, %r27, %r26;
+; SM60-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM60-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM60-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM60-NEXT: min.f32 %r31, %r30, %r29;
+; SM60-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM60-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB299_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<4>;
+; SM60-NEXT: .reg .b32 %r<18>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NEXT: max.f32 %r11, %r9, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p1 bra $L__BB300_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<11>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r10, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: $L__BB301_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: max.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: max.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM60-NEXT: mov.b32 %r10, %r1;
+; SM60-NEXT: @%p1 bra $L__BB301_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM60-NEXT: $L__BB302_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM60-NEXT: max.f32 %r5, %r4, %r3;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM60-NEXT: max.f32 %r8, %r7, %r6;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM60-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM60-NEXT: max.f32 %r12, %r11, %r10;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM60-NEXT: max.f32 %r15, %r14, %r13;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM60-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB302_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<37>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM60-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM60-NEXT: max.f32 %r7, %r6, %r5;
+; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM60-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM60-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM60-NEXT: max.f32 %r10, %r9, %r8;
+; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM60-NEXT: max.f32 %r14, %r13, %r12;
+; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM60-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM60-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM60-NEXT: max.f32 %r17, %r16, %r15;
+; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM60-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB303_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM60-NEXT: $L__BB303_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM60-NEXT: max.f32 %r21, %r20, %r19;
+; SM60-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM60-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM60-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM60-NEXT: max.f32 %r24, %r23, %r22;
+; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM60-NEXT: max.f32 %r28, %r27, %r26;
+; SM60-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM60-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM60-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM60-NEXT: max.f32 %r31, %r30, %r29;
+; SM60-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM60-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM60-NEXT: @%p2 bra $L__BB303_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p6 bra $L__BB304_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: $L__BB305_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p11 bra $L__BB305_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: $L__BB306_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB306_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: $L__BB307_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB307_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB307_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM60-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs33, -32768;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs31, -32768;
+; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM60-NEXT: mov.b16 %rs38, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM60-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs32, -32768;
+; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs30, -32768;
+; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM60-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs48, -32768;
+; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs46, -32768;
+; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM60-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs47, -32768;
+; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs45, -32768;
+; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB307_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: mov.b32 %r14, %r3;
+; SM60-NEXT: @%p6 bra $L__BB308_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: $L__BB309_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: mov.b32 %r4, %r1;
+; SM60-NEXT: @%p11 bra $L__BB309_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: $L__BB310_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB310_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: $L__BB311_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: mov.b16 %rs9, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB311_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM60-NEXT: $L__BB311_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM60-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs33, 0;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs31, 0;
+; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM60-NEXT: mov.b16 %rs38, 0x0000;
+; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM60-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs32, 0;
+; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs30, 0;
+; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM60-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs48, 0;
+; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs46, 0;
+; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM60-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs47, 0;
+; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs45, 0;
+; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB311_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB312_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB312_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB313_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB313_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB314_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: add.rn.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: add.rn.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB314_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB315_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: add.rn.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: add.rn.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: add.rn.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: add.rn.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB315_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB315_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: add.rn.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: add.rn.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: add.rn.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB315_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB316_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB317_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB317_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB318_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: sub.rn.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: sub.rn.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB318_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB319_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: sub.rn.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: sub.rn.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: sub.rn.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: sub.rn.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB319_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB319_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: sub.rn.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: sub.rn.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: sub.rn.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: sub.rn.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB319_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: min.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB320_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB321_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: min.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: min.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB321_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB322_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: min.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: min.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: min.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: min.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB322_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB323_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: min.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: min.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: min.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: min.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB323_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB323_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: min.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: min.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: min.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: min.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB323_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b16 %rs<2>;
+; SM60-NEXT: .reg .b32 %r<24>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: max.f32 %r12, %r9, %r11;
+; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NEXT: mov.b32 %r23, %r3;
+; SM60-NEXT: @%p2 bra $L__BB324_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<4>;
+; SM60-NEXT: .reg .b16 %rs<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r24, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: $L__BB325_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: max.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: max.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM60-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM60-NEXT: mov.b32 %r24, %r1;
+; SM60-NEXT: @%p3 bra $L__BB325_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<47>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: $L__BB326_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: max.f32 %r7, %r6, %r4;
+; SM60-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM60-NEXT: add.s32 %r9, %r8, %r7;
+; SM60-NEXT: add.s32 %r10, %r9, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM60-NEXT: or.b32 %r11, %r7, 4194304;
+; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: max.f32 %r17, %r16, %r14;
+; SM60-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM60-NEXT: add.s32 %r19, %r18, %r17;
+; SM60-NEXT: add.s32 %r20, %r19, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM60-NEXT: or.b32 %r21, %r17, 4194304;
+; SM60-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM60-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: max.f32 %r28, %r27, %r25;
+; SM60-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM60-NEXT: add.s32 %r30, %r29, %r28;
+; SM60-NEXT: add.s32 %r31, %r30, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM60-NEXT: or.b32 %r32, %r28, 4194304;
+; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: max.f32 %r38, %r37, %r35;
+; SM60-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM60-NEXT: add.s32 %r40, %r39, %r38;
+; SM60-NEXT: add.s32 %r41, %r40, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM60-NEXT: or.b32 %r42, %r38, 4194304;
+; SM60-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM60-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB326_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<11>;
+; SM60-NEXT: .reg .b16 %rs<17>;
+; SM60-NEXT: .reg .b32 %r<93>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: $L__BB327_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: max.f32 %r9, %r8, %r6;
+; SM60-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM60-NEXT: add.s32 %r11, %r10, %r9;
+; SM60-NEXT: add.s32 %r12, %r11, 32767;
+; SM60-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM60-NEXT: or.b32 %r13, %r9, 4194304;
+; SM60-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM60-NEXT: shl.b32 %r18, %r17, 16;
+; SM60-NEXT: max.f32 %r19, %r18, %r16;
+; SM60-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM60-NEXT: add.s32 %r21, %r20, %r19;
+; SM60-NEXT: add.s32 %r22, %r21, 32767;
+; SM60-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM60-NEXT: or.b32 %r23, %r19, 4194304;
+; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: max.f32 %r30, %r29, %r27;
+; SM60-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM60-NEXT: add.s32 %r32, %r31, %r30;
+; SM60-NEXT: add.s32 %r33, %r32, 32767;
+; SM60-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM60-NEXT: or.b32 %r34, %r30, 4194304;
+; SM60-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: max.f32 %r40, %r39, %r37;
+; SM60-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM60-NEXT: add.s32 %r42, %r41, %r40;
+; SM60-NEXT: add.s32 %r43, %r42, 32767;
+; SM60-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM60-NEXT: or.b32 %r44, %r40, 4194304;
+; SM60-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM60-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM60-NEXT: @%p5 bra $L__BB327_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM60-NEXT: $L__BB327_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM60-NEXT: shl.b32 %r48, %r47, 16;
+; SM60-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM60-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM60-NEXT: shl.b32 %r50, %r49, 16;
+; SM60-NEXT: max.f32 %r51, %r50, %r48;
+; SM60-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM60-NEXT: add.s32 %r53, %r52, %r51;
+; SM60-NEXT: add.s32 %r54, %r53, 32767;
+; SM60-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM60-NEXT: or.b32 %r55, %r51, 4194304;
+; SM60-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM60-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM60-NEXT: shl.b32 %r58, %r57, 16;
+; SM60-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM60-NEXT: shl.b32 %r60, %r59, 16;
+; SM60-NEXT: max.f32 %r61, %r60, %r58;
+; SM60-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM60-NEXT: add.s32 %r63, %r62, %r61;
+; SM60-NEXT: add.s32 %r64, %r63, 32767;
+; SM60-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM60-NEXT: or.b32 %r65, %r61, 4194304;
+; SM60-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM60-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM60-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM60-NEXT: shl.b32 %r69, %r68, 16;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM60-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM60-NEXT: shl.b32 %r71, %r70, 16;
+; SM60-NEXT: max.f32 %r72, %r71, %r69;
+; SM60-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM60-NEXT: add.s32 %r74, %r73, %r72;
+; SM60-NEXT: add.s32 %r75, %r74, 32767;
+; SM60-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM60-NEXT: or.b32 %r76, %r72, 4194304;
+; SM60-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM60-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM60-NEXT: shl.b32 %r79, %r78, 16;
+; SM60-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM60-NEXT: shl.b32 %r81, %r80, 16;
+; SM60-NEXT: max.f32 %r82, %r81, %r79;
+; SM60-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM60-NEXT: add.s32 %r84, %r83, %r82;
+; SM60-NEXT: add.s32 %r85, %r84, 32767;
+; SM60-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM60-NEXT: or.b32 %r86, %r82, 4194304;
+; SM60-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM60-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM60-NEXT: @%p10 bra $L__BB327_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<8>;
+; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NEXT: mov.b32 %r19, %r3;
+; SM60-NEXT: @%p6 bra $L__BB328_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: $L__BB329_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM60-NEXT: mov.b32 %r16, %r1;
+; SM60-NEXT: @%p11 bra $L__BB329_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<29>;
+; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM60-NEXT: $L__BB330_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.lt.f32 %p11, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: setp.lt.f32 %p16, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB330_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<57>;
+; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM60-NEXT: $L__BB331_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.lt.f32 %p1, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.lt.f32 %p11, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.lt.f32 %p16, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB331_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: $L__BB331_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM60-NEXT: shl.b32 %r32, %r31, 16;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM60-NEXT: shl.b32 %r34, %r33, 16;
+; SM60-NEXT: setp.lt.f32 %p22, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs32, -32768;
+; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs30, -32768;
+; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM60-NEXT: shl.b32 %r36, %r35, 16;
+; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM60-NEXT: shl.b32 %r38, %r37, 16;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM60-NEXT: shl.b32 %r40, %r39, 16;
+; SM60-NEXT: setp.lt.f32 %p27, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs31, -32768;
+; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs29, -32768;
+; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM60-NEXT: shl.b32 %r42, %r41, 16;
+; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM60-NEXT: shl.b32 %r45, %r44, 16;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM60-NEXT: shl.b32 %r47, %r46, 16;
+; SM60-NEXT: setp.lt.f32 %p32, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs46, -32768;
+; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs44, -32768;
+; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM60-NEXT: shl.b32 %r49, %r48, 16;
+; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM60-NEXT: shl.b32 %r51, %r50, 16;
+; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM60-NEXT: shl.b32 %r53, %r52, 16;
+; SM60-NEXT: setp.lt.f32 %p37, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs45, -32768;
+; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs43, -32768;
+; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM60-NEXT: shl.b32 %r55, %r54, 16;
+; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB331_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<7>;
+; SM60-NEXT: .reg .b16 %rs<8>;
+; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NEXT: mov.b32 %r6, 65535;
+; SM60-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NEXT: not.b32 %r2, %r7;
+; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NEXT: mov.b32 %r19, %r3;
+; SM60-NEXT: @%p6 bra $L__BB332_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<12>;
+; SM60-NEXT: .reg .b16 %rs<15>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: $L__BB333_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM60-NEXT: mov.b32 %r16, %r1;
+; SM60-NEXT: @%p11 bra $L__BB333_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<22>;
+; SM60-NEXT: .reg .b16 %rs<29>;
+; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM60-NEXT: shl.b32 %r4, %r3, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM60-NEXT: $L__BB334_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.gt.f32 %p11, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: setp.gt.f32 %p16, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB334_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM60-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<43>;
+; SM60-NEXT: .reg .b16 %rs<57>;
+; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .b64 %rd<22>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
+; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM60-NEXT: $L__BB335_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: setp.gt.f32 %p1, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: shl.b32 %r12, %r11, 16;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM60-NEXT: shl.b32 %r16, %r15, 16;
+; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: shl.b32 %r21, %r20, 16;
+; SM60-NEXT: setp.gt.f32 %p11, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM60-NEXT: shl.b32 %r23, %r22, 16;
+; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM60-NEXT: shl.b32 %r25, %r24, 16;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
+; SM60-NEXT: setp.gt.f32 %p16, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM60-NEXT: shl.b32 %r29, %r28, 16;
+; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM60-NEXT: @%p21 bra $L__BB335_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: $L__BB335_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM60-NEXT: shl.b32 %r32, %r31, 16;
+; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM60-NEXT: shl.b32 %r34, %r33, 16;
+; SM60-NEXT: setp.gt.f32 %p22, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM60-NEXT: setp.eq.b16 %p24, %rs32, 0;
+; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM60-NEXT: setp.eq.b16 %p25, %rs30, 0;
+; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM60-NEXT: shl.b32 %r36, %r35, 16;
+; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM60-NEXT: shl.b32 %r38, %r37, 16;
+; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM60-NEXT: shl.b32 %r40, %r39, 16;
+; SM60-NEXT: setp.gt.f32 %p27, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM60-NEXT: setp.eq.b16 %p29, %rs31, 0;
+; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs29, 0;
+; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM60-NEXT: shl.b32 %r42, %r41, 16;
+; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM60-NEXT: shl.b32 %r45, %r44, 16;
+; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM60-NEXT: shl.b32 %r47, %r46, 16;
+; SM60-NEXT: setp.gt.f32 %p32, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM60-NEXT: setp.eq.b16 %p34, %rs46, 0;
+; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM60-NEXT: setp.eq.b16 %p35, %rs44, 0;
+; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM60-NEXT: shl.b32 %r49, %r48, 16;
+; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM60-NEXT: shl.b32 %r51, %r50, 16;
+; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM60-NEXT: shl.b32 %r53, %r52, 16;
+; SM60-NEXT: setp.gt.f32 %p37, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM60-NEXT: setp.eq.b16 %p39, %rs45, 0;
+; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM60-NEXT: setp.eq.b16 %p40, %rs43, 0;
+; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM60-NEXT: shl.b32 %r55, %r54, 16;
+; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM60-NEXT: @%p42 bra $L__BB335_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_monotonic_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB336_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB336_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_monotonic_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB337_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB337_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_monotonic_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB338_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB338_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_monotonic_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB339_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB339_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_acquire_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB340_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB340_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_acquire_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB341_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB341_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_acquire_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB342_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB342_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_acquire_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB343_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB343_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_release_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB344_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB344_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_release_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB345_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_release_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB346_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB346_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_release_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB347_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB347_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: $L__BB348_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: add.s32 %r10, %r15, %r4;
+; SM60-NEXT: and.b32 %r11, %r10, %r2;
+; SM60-NEXT: and.b32 %r12, %r15, %r3;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM60-NEXT: mov.b32 %r15, %r5;
+; SM60-NEXT: @%p1 bra $L__BB348_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<12>;
+; SM60-NEXT: .reg .b32 %r<16>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM60-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r15, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM60-NEXT: and.b16 %rs9, %rs8, 255;
+; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM60-NEXT: shl.b32 %r11, %r10, %r1;
+; SM60-NEXT: and.b32 %r12, %r15, %r2;
+; SM60-NEXT: or.b32 %r13, %r12, %r11;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM60-NEXT: mov.b32 %r15, %r3;
+; SM60-NEXT: @%p1 bra $L__BB349_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r14, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r14;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: $L__BB350_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM60-NEXT: mov.b32 %r18, %r1;
+; SM60-NEXT: @%p1 bra $L__BB350_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: add_seq_cst_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<35>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM60-NEXT: $L__BB351_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB351_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_monotonic_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_monotonic_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_monotonic_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_monotonic_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_acquire_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_acquire_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_acquire_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_acquire_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_release_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_release_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_release_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<3>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<5>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: add_seq_cst_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB368_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB369_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB370_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_monotonic_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB371_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_acquire_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB372_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_acquire_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB373_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_acquire_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB374_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_acquire_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB375_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_release_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB376_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_release_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB377_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB377_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_release_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB378_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB378_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_release_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB379_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB379_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v1i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: and.b32 %r8, %r7, 3;
+; SM60-NEXT: shl.b32 %r1, %r8, 3;
+; SM60-NEXT: mov.b32 %r9, 255;
+; SM60-NEXT: shl.b32 %r2, %r9, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: shl.b32 %r4, %r6, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: $L__BB380_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
+; SM60-NEXT: @%p1 bra $L__BB380_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b8 [func_retval0], %r15;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b64 %rd<3>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: and.b32 %r6, %r5, 3;
+; SM60-NEXT: shl.b32 %r1, %r6, 3;
+; SM60-NEXT: mov.b32 %r7, 65535;
+; SM60-NEXT: shl.b32 %r8, %r7, %r1;
+; SM60-NEXT: not.b32 %r2, %r8;
+; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: $L__BB381_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: shr.u32 %r9, %r18, %r1;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM60-NEXT: and.b32 %r11, %r10, %r4;
+; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM60-NEXT: shl.b32 %r14, %r13, %r1;
+; SM60-NEXT: and.b32 %r15, %r18, %r2;
+; SM60-NEXT: or.b32 %r16, %r15, %r14;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: @%p1 bra $L__BB381_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB382_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r2;
+; SM60-NEXT: xor.b32 %r4, %r3, -1;
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM60-NEXT: mov.b32 %r5, %r1;
+; SM60-NEXT: @%p1 bra $L__BB382_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .b64 %rd<12>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM60-NEXT: $L__BB383_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r8, %r2;
+; SM60-NEXT: and.b32 %r4, %r7, %r1;
+; SM60-NEXT: xor.b32 %r5, %r4, -1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM60-NEXT: xor.b32 %r6, %r3, -1;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM60-NEXT: @%p1 bra $L__BB383_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB384_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB385_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB386_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB386_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB386_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_monotonic_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB387_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB387_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB387_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB387_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB387_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB387_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB387_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_acquire_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB388_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_acquire_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB389_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_acquire_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB390_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB390_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_acquire_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB391_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB391_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB391_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB391_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB391_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB391_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB391_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_release_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB392_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_release_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB393_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_release_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB394_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB394_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_release_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB395_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB395_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB395_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB395_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB395_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB395_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB395_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<6>;
+; SM60-NEXT: .reg .b64 %rd<2>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
+; SM60-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: and.b32 %r3, %r5, %r1;
+; SM60-NEXT: not.b32 %r4, %r3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM60-NEXT: mov.b32 %r5, %r2;
+; SM60-NEXT: @%p1 bra $L__BB396_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<2>;
+; SM60-NEXT: .reg .b32 %r<7>;
+; SM60-NEXT: .reg .b64 %rd<13>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM60-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r3, %r6, %r2;
+; SM60-NEXT: and.b32 %r4, %r5, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB397_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<3>;
+; SM60-NEXT: .reg .b32 %r<13>;
+; SM60-NEXT: .reg .b64 %rd<24>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM60-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r5, %r10, %r2;
+; SM60-NEXT: and.b32 %r6, %r9, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB398_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM60-NEXT: $L__BB398_3: // %atomicrmw.start2
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r7, %r12, %r4;
+; SM60-NEXT: and.b32 %r8, %r11, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB398_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end1
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM60-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM60: {
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<25>;
+; SM60-NEXT: .reg .b64 %rd<46>;
+; SM60-EMPTY:
+; SM60-NEXT: // %bb.0:
+; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM60-NEXT: $L__BB399_1: // %atomicrmw.start
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: and.b32 %r9, %r18, %r6;
+; SM60-NEXT: and.b32 %r10, %r17, %r5;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM60-NEXT: not.b64 %rd11, %rd10;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM60-NEXT: @%p1 bra $L__BB399_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM60-NEXT: $L__BB399_3: // %atomicrmw.start6
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM60-NEXT: and.b32 %r11, %r20, %r8;
+; SM60-NEXT: and.b32 %r12, %r19, %r7;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM60-NEXT: not.b64 %rd22, %rd21;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM60-NEXT: @%p2 bra $L__BB399_3;
+; SM60-NEXT: // %bb.4: // %atomicrmw.end5
+; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM60-NEXT: $L__BB399_5: // %atomicrmw.start16
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM60-NEXT: and.b32 %r13, %r22, %r2;
+; SM60-NEXT: and.b32 %r14, %r21, %r1;
+; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM60-NEXT: not.b64 %rd33, %rd32;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM60-NEXT: @%p3 bra $L__BB399_5;
+; SM60-NEXT: // %bb.6: // %atomicrmw.end15
+; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM60-NEXT: $L__BB399_7: // %atomicrmw.start22
+; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM60-NEXT: and.b32 %r15, %r24, %r4;
+; SM60-NEXT: and.b32 %r16, %r23, %r3;
+; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM60-NEXT: not.b64 %rd44, %rd43;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM60-NEXT: @%p4 bra $L__BB399_7;
+; SM60-NEXT: // %bb.8: // %atomicrmw.end21
+; SM60-NEXT: membar.cta;
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM60-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
new file mode 100644
index 0000000000000..4a4589fe80f56
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -0,0 +1,19233 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefix=SM70
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 255;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB0_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB1_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<14>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r9, %r8, %r1;
+; SM70-NEXT: not.b32 %r2, %r9;
+; SM70-NEXT: shl.b32 %r3, %r5, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM70-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r13, %r2;
+; SM70-NEXT: or.b32 %r11, %r10, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM70-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM70-NEXT: mov.b32 %r13, %r4;
+; SM70-NEXT: @%p1 bra $L__BB4_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r12, %r4, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: cvt.u64.u32 %rd2, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r2;
+; SM70-NEXT: shl.b64 %rd4, %rd3, 32;
+; SM70-NEXT: or.b64 %rd5, %rd2, %rd4;
+; SM70-NEXT: cvt.u64.u32 %rd6, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: shl.b64 %rd8, %rd7, 32;
+; SM70-NEXT: or.b64 %rd9, %rd6, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB16_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB17_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB18_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB18_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB19_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB19_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB20_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB21_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB21_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB22_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB22_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: add_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB23_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: add.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: add.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: add.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: add.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB23_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB23_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: add.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: add.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: add.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: add.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB23_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: add_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB32_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB33_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB33_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB34_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB34_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB35_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB35_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB36_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB37_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB38_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB39_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB39_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB39_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r2, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r3, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM70-NEXT: neg.s32 %r5, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r5, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM70-NEXT: neg.s32 %r7, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM70-NEXT: neg.s32 %r9, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM70-NEXT: neg.s32 %r11, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: neg.s32 %r9, %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM70-NEXT: neg.s32 %r11, %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM70-NEXT: neg.s32 %r13, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM70-NEXT: neg.s32 %r15, %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM70-NEXT: neg.s32 %r17, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM70-NEXT: neg.s32 %r19, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM70-NEXT: neg.s32 %r21, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM70-NEXT: neg.s32 %r23, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<5>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd3, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<8>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd4, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM70-NEXT: neg.s64 %rd6, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<14>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM70-NEXT: neg.s64 %rd8, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM70-NEXT: neg.s64 %rd10, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM70-NEXT: neg.s64 %rd12, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<26>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: neg.s64 %rd10, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM70-NEXT: neg.s64 %rd12, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM70-NEXT: neg.s64 %rd14, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM70-NEXT: neg.s64 %rd16, %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM70-NEXT: neg.s64 %rd18, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM70-NEXT: neg.s64 %rd20, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM70-NEXT: neg.s64 %rd22, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM70-NEXT: neg.s64 %rd24, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: mov.b32 %r5, 255;
+; SM70-NEXT: shl.b32 %r6, %r5, %r4;
+; SM70-NEXT: not.b32 %r7, %r6;
+; SM70-NEXT: shl.b32 %r8, %r1, %r4;
+; SM70-NEXT: or.b32 %r9, %r8, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: mov.b32 %r4, 65535;
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: not.b32 %r6, %r5;
+; SM70-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r8, %r7, %r3;
+; SM70-NEXT: or.b32 %r9, %r8, %r6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB50_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: and_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: and.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB51_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: mov.b32 %r5, 65535;
+; SM70-NEXT: shl.b32 %r6, %r5, %r4;
+; SM70-NEXT: not.b32 %r7, %r6;
+; SM70-NEXT: shl.b32 %r8, %r1, %r4;
+; SM70-NEXT: or.b32 %r9, %r8, %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM70-NEXT: shr.u32 %r11, %r10, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB53_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: and.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB54_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: and_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: and.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB55_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB55_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: and.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB55_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: and_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: and_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB64_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB65_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB66_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB66_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB67_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB67_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 65535;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB68_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB69_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB69_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB70_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB70_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB71_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r14, %r2;
+; SM70-NEXT: and.b32 %r6, %r13, %r1;
+; SM70-NEXT: xor.b32 %r7, %r6, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: xor.b32 %r8, %r5, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB71_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB71_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: and.b32 %r9, %r16, %r4;
+; SM70-NEXT: and.b32 %r10, %r15, %r3;
+; SM70-NEXT: xor.b32 %r11, %r10, -1;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: xor.b32 %r12, %r9, -1;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB71_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB72_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB74_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB74_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB75_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB75_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB75_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB75_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB75_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB75_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM70-NEXT: not.b64 %rd5, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB76_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd6, %rd10, %rd1;
+; SM70-NEXT: not.b64 %rd7, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB77_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB77_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd8, %rd11, %rd2;
+; SM70-NEXT: not.b64 %rd9, %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB77_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: and.b64 %rd10, %rd5, %rd3;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB78_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB78_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: and.b64 %rd12, %rd6, %rd4;
+; SM70-NEXT: not.b64 %rd13, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB78_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB78_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd14, %rd20, %rd1;
+; SM70-NEXT: not.b64 %rd15, %rd14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB78_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB78_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd16, %rd21, %rd2;
+; SM70-NEXT: not.b64 %rd17, %rd16;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB78_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: and.b64 %rd18, %rd7, %rd5;
+; SM70-NEXT: not.b64 %rd19, %rd18;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB79_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB79_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: and.b64 %rd20, %rd8, %rd6;
+; SM70-NEXT: not.b64 %rd21, %rd20;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB79_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB79_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: and.b64 %rd22, %rd9, %rd3;
+; SM70-NEXT: not.b64 %rd23, %rd22;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB79_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB79_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: and.b64 %rd24, %rd10, %rd4;
+; SM70-NEXT: not.b64 %rd25, %rd24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB79_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB79_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: and.b64 %rd26, %rd13, %rd1;
+; SM70-NEXT: not.b64 %rd27, %rd26;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB79_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB79_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: and.b64 %rd28, %rd14, %rd2;
+; SM70-NEXT: not.b64 %rd29, %rd28;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB79_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB79_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd30, %rd40, %rd11;
+; SM70-NEXT: not.b64 %rd31, %rd30;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB79_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB79_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b64 %rd32, %rd41, %rd12;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB79_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB82_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: or_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: or.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB83_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB85_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: or.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB86_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: or_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: or.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB87_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB87_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: or.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: or.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB87_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: or_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: or_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, 3;
+; SM70-NEXT: shl.b32 %r3, %r2, 3;
+; SM70-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r5, %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB98_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: xor.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB99_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd2, %rd1, -4;
+; SM70-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM70-NEXT: and.b32 %r3, %r2, 3;
+; SM70-NEXT: shl.b32 %r4, %r3, 3;
+; SM70-NEXT: shl.b32 %r5, %r1, %r4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM70-NEXT: shr.u32 %r7, %r6, %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB101_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB102_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: xor.b32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB102_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB103_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: xor.b32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB103_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB103_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: xor.b32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: xor.b32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB103_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM70-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB112_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<14>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM70-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r16, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM70-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM70-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM70-NEXT: max.s16 %rs9, %rs4, %rs8;
+; SM70-NEXT: max.s16 %rs10, %rs3, %rs7;
+; SM70-NEXT: and.b16 %rs11, %rs10, 255;
+; SM70-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM70-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM70-NEXT: shl.b32 %r12, %r11, %r1;
+; SM70-NEXT: and.b32 %r13, %r16, %r2;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM70-NEXT: mov.b32 %r16, %r3;
+; SM70-NEXT: @%p1 bra $L__BB113_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM70-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM70-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM70-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM70-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM70-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM70-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM70-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p5 bra $L__BB114_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: max_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<51>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: $L__BB115_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM70-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM70-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM70-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM70-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM70-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM70-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM70-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM70-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM70-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM70-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM70-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM70-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM70-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM70-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM70-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM70-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM70-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM70-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM70-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM70-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB115_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB116_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB117_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB117_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB118_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB118_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: max_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB119_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: max.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: max.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB119_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB119_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: max.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: max.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: max.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: max.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB119_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: max_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: max_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM70-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB128_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<14>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: cvt.s16.s8 %rs7, %rs5;
+; SM70-NEXT: cvt.s16.s8 %rs8, %rs6;
+; SM70-NEXT: $L__BB129_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r16, %r1;
+; SM70-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM70-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM70-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM70-NEXT: min.s16 %rs9, %rs4, %rs8;
+; SM70-NEXT: min.s16 %rs10, %rs3, %rs7;
+; SM70-NEXT: and.b16 %rs11, %rs10, 255;
+; SM70-NEXT: shl.b16 %rs12, %rs9, 8;
+; SM70-NEXT: or.b16 %rs13, %rs11, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs13;
+; SM70-NEXT: shl.b32 %r12, %r11, %r1;
+; SM70-NEXT: and.b32 %r13, %r16, %r2;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r16;
+; SM70-NEXT: mov.b32 %r16, %r3;
+; SM70-NEXT: @%p1 bra $L__BB129_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM70-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM70-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM70-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM70-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM70-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM70-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM70-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p5 bra $L__BB130_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: min_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<51>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM70-NEXT: $L__BB131_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM70-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM70-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM70-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM70-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM70-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM70-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM70-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM70-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM70-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM70-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM70-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM70-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM70-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM70-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM70-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM70-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM70-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM70-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM70-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM70-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM70-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM70-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM70-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM70-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB131_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB132_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB133_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB134_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: min_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: min.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: min.s16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.s16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB135_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB135_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: min.s16 %rs17, %rs16, %rs14;
+; SM70-NEXT: min.s16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: min.s16 %rs23, %rs22, %rs20;
+; SM70-NEXT: min.s16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB135_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: min_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: min_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB144_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: max.u16 %rs9, %rs5, %rs7;
+; SM70-NEXT: max.u16 %rs10, %rs6, %rs8;
+; SM70-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM70-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB145_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM70-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM70-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM70-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM70-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p5 bra $L__BB146_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB147_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB148_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB149_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB150_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB150_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB151_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: max.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: max.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: max.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB151_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB151_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: max.u16 %rs17, %rs16, %rs14;
+; SM70-NEXT: max.u16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: max.u16 %rs23, %rs22, %rs20;
+; SM70-NEXT: max.u16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB151_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB160_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: min.u16 %rs9, %rs5, %rs7;
+; SM70-NEXT: min.u16 %rs10, %rs6, %rs8;
+; SM70-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM70-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB161_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM70-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM70-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM70-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM70-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p5 bra $L__BB162_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB163_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB164_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB165_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM70-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB166_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: min.u16 %rs5, %rs4, %rs2;
+; SM70-NEXT: min.u16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM70-NEXT: min.u16 %rs11, %rs10, %rs8;
+; SM70-NEXT: min.u16 %rs12, %rs9, %rs7;
+; SM70-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB167_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: $L__BB167_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM70-NEXT: min.u16 %rs17, %rs16, %rs14;
+; SM70-NEXT: min.u16 %rs18, %rs15, %rs13;
+; SM70-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: min.u16 %rs23, %rs22, %rs20;
+; SM70-NEXT: min.u16 %rs24, %rs21, %rs19;
+; SM70-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB167_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p2 bra $L__BB176_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM70-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: add.s16 %rs7, %rs3, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs5, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs8;
+; SM70-NEXT: selp.b16 %rs10, 0, %rs7, %p2;
+; SM70-NEXT: selp.b16 %rs11, 0, %rs6, %p1;
+; SM70-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM70-NEXT: and.b16 %rs13, %rs10, 255;
+; SM70-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB177_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<23>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, 1;
+; SM70-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, 1;
+; SM70-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, 1;
+; SM70-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM70-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM70-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM70-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM70-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM70-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM70-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM70-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM70-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM70-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM70-NEXT: mov.b32 %r22, %r1;
+; SM70-NEXT: @%p5 bra $L__BB178_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, 1;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, 1;
+; SM70-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, 1;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, 1;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM70-NEXT: add.s16 %rs10, %rs9, 1;
+; SM70-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM70-NEXT: add.s16 %rs12, %rs11, 1;
+; SM70-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM70-NEXT: add.s16 %rs14, %rs13, 1;
+; SM70-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM70-NEXT: add.s16 %rs16, %rs15, 1;
+; SM70-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM70-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM70-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM70-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM70-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM70-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM70-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM70-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB179_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: add.s16 %rs3, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p2 bra $L__BB180_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: $L__BB181_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs5;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs6;
+; SM70-NEXT: selp.b16 %rs7, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs8, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p3 bra $L__BB181_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB182_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM70-NEXT: add.s16 %rs7, %rs5, 1;
+; SM70-NEXT: add.s16 %rs8, %rs6, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM70-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB182_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM70-NEXT: add.s16 %rs3, %rs1, 1;
+; SM70-NEXT: add.s16 %rs4, %rs2, 1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM70-NEXT: add.s16 %rs7, %rs5, 1;
+; SM70-NEXT: add.s16 %rs8, %rs6, 1;
+; SM70-NEXT: setp.ge.u16 %p1, %rs1, %rs9;
+; SM70-NEXT: setp.ge.u16 %p2, %rs2, %rs10;
+; SM70-NEXT: setp.ge.u16 %p3, %rs5, %rs11;
+; SM70-NEXT: setp.ge.u16 %p4, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, 0, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, 0, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs15, 0, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, 0, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB183_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB183_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r12;
+; SM70-NEXT: add.s16 %rs19, %rs17, 1;
+; SM70-NEXT: add.s16 %rs20, %rs18, 1;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r11;
+; SM70-NEXT: add.s16 %rs23, %rs21, 1;
+; SM70-NEXT: add.s16 %rs24, %rs22, 1;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM70-NEXT: setp.ge.u16 %p6, %rs17, %rs25;
+; SM70-NEXT: setp.ge.u16 %p7, %rs18, %rs26;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r3;
+; SM70-NEXT: setp.ge.u16 %p8, %rs21, %rs27;
+; SM70-NEXT: setp.ge.u16 %p9, %rs22, %rs28;
+; SM70-NEXT: selp.b16 %rs29, 0, %rs24, %p9;
+; SM70-NEXT: selp.b16 %rs30, 0, %rs23, %p8;
+; SM70-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs31, 0, %rs20, %p7;
+; SM70-NEXT: selp.b16 %rs32, 0, %rs19, %p6;
+; SM70-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB183_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd4, %rd6, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM70-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p2 bra $L__BB188_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd6, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM70-NEXT: selp.b64 %rd7, 0, %rd6, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p2 bra $L__BB189_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB189_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd8, %rd11, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM70-NEXT: selp.b64 %rd9, 0, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p4 bra $L__BB189_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: add.s64 %rd10, %rd5, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, 0, %rd10, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM70-NEXT: @%p2 bra $L__BB190_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB190_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: add.s64 %rd12, %rd6, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd13, 0, %rd12, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM70-NEXT: @%p4 bra $L__BB190_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB190_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd14, %rd20, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM70-NEXT: selp.b64 %rd15, 0, %rd14, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p6 bra $L__BB190_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB190_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd16, %rd21, 1;
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM70-NEXT: selp.b64 %rd17, 0, %rd16, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB190_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: add.s64 %rd18, %rd7, 1;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, 0, %rd18, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM70-NEXT: @%p2 bra $L__BB191_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB191_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: add.s64 %rd20, %rd8, 1;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd21, 0, %rd20, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB191_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB191_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: add.s64 %rd22, %rd9, 1;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd23, 0, %rd22, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM70-NEXT: @%p6 bra $L__BB191_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB191_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: add.s64 %rd24, %rd10, 1;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd25, 0, %rd24, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM70-NEXT: @%p8 bra $L__BB191_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB191_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: add.s64 %rd26, %rd13, 1;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd27, 0, %rd26, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM70-NEXT: @%p10 bra $L__BB191_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB191_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: add.s64 %rd28, %rd14, 1;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd29, 0, %rd28, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM70-NEXT: @%p12 bra $L__BB191_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB191_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd30, %rd40, 1;
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM70-NEXT: selp.b64 %rd31, 0, %rd30, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p14 bra $L__BB191_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB191_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd32, %rd41, 1;
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM70-NEXT: selp.b64 %rd33, 0, %rd32, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p16 bra $L__BB191_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM70-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB192_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM70-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: add.s16 %rs7, %rs3, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM70-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM70-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM70-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM70-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM70-NEXT: and.b16 %rs15, %rs11, 255;
+; SM70-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p5 bra $L__BB193_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<27>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM70-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, -1;
+; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, -1;
+; SM70-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, -1;
+; SM70-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM70-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM70-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM70-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM70-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM70-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM70-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM70-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM70-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM70-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM70-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM70-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM70-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM70-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM70-NEXT: mov.b32 %r26, %r1;
+; SM70-NEXT: @%p9 bra $L__BB194_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<41>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM70-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: add.s16 %rs2, %rs1, -1;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM70-NEXT: add.s16 %rs4, %rs3, -1;
+; SM70-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM70-NEXT: add.s16 %rs6, %rs5, -1;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: add.s16 %rs8, %rs7, -1;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM70-NEXT: add.s16 %rs10, %rs9, -1;
+; SM70-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM70-NEXT: add.s16 %rs12, %rs11, -1;
+; SM70-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM70-NEXT: add.s16 %rs14, %rs13, -1;
+; SM70-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM70-NEXT: add.s16 %rs16, %rs15, -1;
+; SM70-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM70-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM70-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM70-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM70-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM70-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM70-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM70-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM70-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM70-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM70-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM70-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM70-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM70-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM70-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM70-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM70-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM70-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM70-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM70-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM70-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM70-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM70-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM70-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB195_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: add.s16 %rs3, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM70-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p3 bra $L__BB196_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<11>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM70-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM70-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM70-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p5 bra $L__BB197_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<21>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB198_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM70-NEXT: add.s16 %rs7, %rs5, -1;
+; SM70-NEXT: add.s16 %rs8, %rs6, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM70-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM70-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM70-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM70-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM70-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM70-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM70-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB198_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b16 %rs<41>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM70-NEXT: $L__BB199_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r10;
+; SM70-NEXT: add.s16 %rs3, %rs1, -1;
+; SM70-NEXT: add.s16 %rs4, %rs2, -1;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r9;
+; SM70-NEXT: add.s16 %rs7, %rs5, -1;
+; SM70-NEXT: add.s16 %rs8, %rs6, -1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM70-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM70-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM70-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM70-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM70-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM70-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM70-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM70-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs16, %rs14};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM70-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM70-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM70-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs20, %rs18};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB199_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB199_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM70-NEXT: add.s16 %rs23, %rs21, -1;
+; SM70-NEXT: add.s16 %rs24, %rs22, -1;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r11;
+; SM70-NEXT: add.s16 %rs27, %rs25, -1;
+; SM70-NEXT: add.s16 %rs28, %rs26, -1;
+; SM70-NEXT: setp.eq.b16 %p10, %rs21, 0;
+; SM70-NEXT: setp.eq.b16 %p11, %rs22, 0;
+; SM70-NEXT: setp.eq.b16 %p12, %rs25, 0;
+; SM70-NEXT: setp.eq.b16 %p13, %rs26, 0;
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r4;
+; SM70-NEXT: setp.gt.u16 %p14, %rs21, %rs29;
+; SM70-NEXT: setp.gt.u16 %p15, %rs22, %rs30;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r3;
+; SM70-NEXT: setp.gt.u16 %p16, %rs25, %rs31;
+; SM70-NEXT: setp.gt.u16 %p17, %rs26, %rs32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs28, %p17;
+; SM70-NEXT: selp.b16 %rs34, %rs32, %rs33, %p13;
+; SM70-NEXT: selp.b16 %rs35, %rs31, %rs27, %p16;
+; SM70-NEXT: selp.b16 %rs36, %rs31, %rs35, %p12;
+; SM70-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs37, %rs30, %rs24, %p15;
+; SM70-NEXT: selp.b16 %rs38, %rs30, %rs37, %p11;
+; SM70-NEXT: selp.b16 %rs39, %rs29, %rs23, %p14;
+; SM70-NEXT: selp.b16 %rs40, %rs29, %rs39, %p10;
+; SM70-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB199_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b64 %rd<8>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM70-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd4, %rd7, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM70-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM70-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM70-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM70-NEXT: mov.b64 %rd7, %rd2;
+; SM70-NEXT: @%p3 bra $L__BB204_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b64 %rd<14>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd5];
+; SM70-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd6, %rd12, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd12, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd12, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd1, %rd6, %p2;
+; SM70-NEXT: selp.b64 %rd8, %rd1, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd12, %rd8;
+; SM70-NEXT: setp.ne.b64 %p3, %rd3, %rd12;
+; SM70-NEXT: mov.b64 %rd12, %rd3;
+; SM70-NEXT: @%p3 bra $L__BB205_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd5+8];
+; SM70-NEXT: $L__BB205_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd9, %rd13, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd13, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd13, %rd2;
+; SM70-NEXT: selp.b64 %rd10, %rd2, %rd9, %p5;
+; SM70-NEXT: selp.b64 %rd11, %rd2, %rd10, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
+; SM70-NEXT: setp.ne.b64 %p6, %rd4, %rd13;
+; SM70-NEXT: mov.b64 %rd13, %rd4;
+; SM70-NEXT: @%p6 bra $L__BB205_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<13>;
+; SM70-NEXT: .reg .b64 %rd<26>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd9];
+; SM70-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd22;
+; SM70-NEXT: add.s64 %rd10, %rd5, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, %rd3, %rd10, %p2;
+; SM70-NEXT: selp.b64 %rd12, %rd3, %rd11, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd5, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd22, %rd5;
+; SM70-NEXT: @%p3 bra $L__BB206_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd23, [%rd9+8];
+; SM70-NEXT: $L__BB206_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd23;
+; SM70-NEXT: add.s64 %rd13, %rd6, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd6, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd14, %rd4, %rd13, %p5;
+; SM70-NEXT: selp.b64 %rd15, %rd4, %rd14, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd6, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd23, %rd6;
+; SM70-NEXT: @%p6 bra $L__BB206_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd9+16];
+; SM70-NEXT: $L__BB206_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd16, %rd24, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd24, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd24, %rd1;
+; SM70-NEXT: selp.b64 %rd17, %rd1, %rd16, %p8;
+; SM70-NEXT: selp.b64 %rd18, %rd1, %rd17, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd24, %rd18;
+; SM70-NEXT: setp.ne.b64 %p9, %rd7, %rd24;
+; SM70-NEXT: mov.b64 %rd24, %rd7;
+; SM70-NEXT: @%p9 bra $L__BB206_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd25, [%rd9+24];
+; SM70-NEXT: $L__BB206_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd19, %rd25, -1;
+; SM70-NEXT: setp.eq.b64 %p10, %rd25, 0;
+; SM70-NEXT: setp.gt.u64 %p11, %rd25, %rd2;
+; SM70-NEXT: selp.b64 %rd20, %rd2, %rd19, %p11;
+; SM70-NEXT: selp.b64 %rd21, %rd2, %rd20, %p10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd8, %rd25;
+; SM70-NEXT: mov.b64 %rd25, %rd8;
+; SM70-NEXT: @%p12 bra $L__BB206_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd22, %rd23};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<25>;
+; SM70-NEXT: .reg .b64 %rd<50>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd42, [%rd17];
+; SM70-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd42;
+; SM70-NEXT: add.s64 %rd18, %rd7, -1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: setp.gt.u64 %p2, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd5, %rd18, %p2;
+; SM70-NEXT: selp.b64 %rd20, %rd5, %rd19, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd7, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd42, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB207_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd43, [%rd17+8];
+; SM70-NEXT: $L__BB207_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd43;
+; SM70-NEXT: add.s64 %rd21, %rd8, -1;
+; SM70-NEXT: setp.eq.b64 %p4, %rd8, 0;
+; SM70-NEXT: setp.gt.u64 %p5, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd22, %rd6, %rd21, %p5;
+; SM70-NEXT: selp.b64 %rd23, %rd6, %rd22, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd8, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd43, %rd8;
+; SM70-NEXT: @%p6 bra $L__BB207_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd44, [%rd17+16];
+; SM70-NEXT: $L__BB207_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd44;
+; SM70-NEXT: add.s64 %rd24, %rd9, -1;
+; SM70-NEXT: setp.eq.b64 %p7, %rd9, 0;
+; SM70-NEXT: setp.gt.u64 %p8, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd25, %rd3, %rd24, %p8;
+; SM70-NEXT: selp.b64 %rd26, %rd3, %rd25, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd9, %rd26;
+; SM70-NEXT: setp.ne.b64 %p9, %rd44, %rd9;
+; SM70-NEXT: @%p9 bra $L__BB207_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd45, [%rd17+24];
+; SM70-NEXT: $L__BB207_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd45;
+; SM70-NEXT: add.s64 %rd27, %rd10, -1;
+; SM70-NEXT: setp.eq.b64 %p10, %rd10, 0;
+; SM70-NEXT: setp.gt.u64 %p11, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd28, %rd4, %rd27, %p11;
+; SM70-NEXT: selp.b64 %rd29, %rd4, %rd28, %p10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd10, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd45, %rd10;
+; SM70-NEXT: @%p12 bra $L__BB207_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd46, [%rd17+32];
+; SM70-NEXT: $L__BB207_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd46;
+; SM70-NEXT: add.s64 %rd30, %rd13, -1;
+; SM70-NEXT: setp.eq.b64 %p13, %rd13, 0;
+; SM70-NEXT: setp.gt.u64 %p14, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd31, %rd1, %rd30, %p14;
+; SM70-NEXT: selp.b64 %rd32, %rd1, %rd31, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd13, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd46, %rd13;
+; SM70-NEXT: @%p15 bra $L__BB207_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd47, [%rd17+40];
+; SM70-NEXT: $L__BB207_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd47;
+; SM70-NEXT: add.s64 %rd33, %rd14, -1;
+; SM70-NEXT: setp.eq.b64 %p16, %rd14, 0;
+; SM70-NEXT: setp.gt.u64 %p17, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd34, %rd2, %rd33, %p17;
+; SM70-NEXT: selp.b64 %rd35, %rd2, %rd34, %p16;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd14, %rd35;
+; SM70-NEXT: setp.ne.b64 %p18, %rd47, %rd14;
+; SM70-NEXT: @%p18 bra $L__BB207_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd48, [%rd17+48];
+; SM70-NEXT: $L__BB207_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd36, %rd48, -1;
+; SM70-NEXT: setp.eq.b64 %p19, %rd48, 0;
+; SM70-NEXT: setp.gt.u64 %p20, %rd48, %rd11;
+; SM70-NEXT: selp.b64 %rd37, %rd11, %rd36, %p20;
+; SM70-NEXT: selp.b64 %rd38, %rd11, %rd37, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd48, %rd38;
+; SM70-NEXT: setp.ne.b64 %p21, %rd15, %rd48;
+; SM70-NEXT: mov.b64 %rd48, %rd15;
+; SM70-NEXT: @%p21 bra $L__BB207_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd49, [%rd17+56];
+; SM70-NEXT: $L__BB207_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s64 %rd39, %rd49, -1;
+; SM70-NEXT: setp.eq.b64 %p22, %rd49, 0;
+; SM70-NEXT: setp.gt.u64 %p23, %rd49, %rd12;
+; SM70-NEXT: selp.b64 %rd40, %rd12, %rd39, %p23;
+; SM70-NEXT: selp.b64 %rd41, %rd12, %rd40, %p22;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd16, %rd49;
+; SM70-NEXT: mov.b64 %rd49, %rd16;
+; SM70-NEXT: @%p24 bra $L__BB207_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: and.b32 %r10, %r9, 255;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p2 bra $L__BB208_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM70-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: and.b16 %rs4, %rs3, 255;
+; SM70-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM70-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM70-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM70-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM70-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM70-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM70-NEXT: and.b16 %rs13, %rs10, 255;
+; SM70-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p3 bra $L__BB209_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<23>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM70-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM70-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM70-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM70-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM70-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM70-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM70-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM70-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM70-NEXT: mov.b32 %r22, %r1;
+; SM70-NEXT: @%p5 bra $L__BB210_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM70-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM70-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM70-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM70-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM70-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM70-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM70-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM70-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM70-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM70-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM70-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM70-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM70-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM70-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM70-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM70-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM70-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM70-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM70-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM70-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM70-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM70-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM70-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM70-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB211_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p2 bra $L__BB212_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p3 bra $L__BB213_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM70-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM70-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM70-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM70-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB214_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM70-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r9;
+; SM70-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM70-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM70-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM70-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM70-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM70-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM70-NEXT: mov.b32 %r5, {%rs14, %rs13};
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM70-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM70-NEXT: mov.b32 %r6, {%rs16, %rs15};
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB215_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB215_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r12;
+; SM70-NEXT: setp.ge.u16 %p6, %rs19, %rs17;
+; SM70-NEXT: setp.ge.u16 %p7, %rs20, %rs18;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r3;
+; SM70-NEXT: mov.b32 {%rs23, %rs24}, %r11;
+; SM70-NEXT: setp.ge.u16 %p8, %rs23, %rs21;
+; SM70-NEXT: setp.ge.u16 %p9, %rs24, %rs22;
+; SM70-NEXT: sub.s16 %rs25, %rs19, %rs17;
+; SM70-NEXT: sub.s16 %rs26, %rs20, %rs18;
+; SM70-NEXT: sub.s16 %rs27, %rs23, %rs21;
+; SM70-NEXT: sub.s16 %rs28, %rs24, %rs22;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs24, %p9;
+; SM70-NEXT: selp.b16 %rs30, %rs27, %rs23, %p8;
+; SM70-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM70-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM70-NEXT: selp.b16 %rs31, %rs26, %rs20, %p7;
+; SM70-NEXT: selp.b16 %rs32, %rs25, %rs19, %p6;
+; SM70-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB215_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM70-NEXT: sub.s32 %r3, %r5, %r1;
+; SM70-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p2 bra $L__BB216_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB217_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM70-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM70-NEXT: sub.s32 %r3, %r8, %r2;
+; SM70-NEXT: sub.s32 %r4, %r7, %r1;
+; SM70-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB217_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB218_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r14, %r2;
+; SM70-NEXT: setp.ge.u32 %p2, %r13, %r1;
+; SM70-NEXT: sub.s32 %r5, %r14, %r2;
+; SM70-NEXT: sub.s32 %r6, %r13, %r1;
+; SM70-NEXT: selp.b32 %r7, %r6, %r13, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: selp.b32 %r8, %r5, %r14, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB218_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB218_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: setp.ge.u32 %p4, %r16, %r4;
+; SM70-NEXT: setp.ge.u32 %p5, %r15, %r3;
+; SM70-NEXT: sub.s32 %r9, %r16, %r4;
+; SM70-NEXT: sub.s32 %r10, %r15, %r3;
+; SM70-NEXT: selp.b32 %r11, %r10, %r15, %p5;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: selp.b32 %r12, %r9, %r16, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p6 bra $L__BB218_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<13>;
+; SM70-NEXT: .reg .b32 %r<33>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r26;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: setp.ge.u32 %p1, %r26, %r6;
+; SM70-NEXT: setp.ge.u32 %p2, %r25, %r5;
+; SM70-NEXT: sub.s32 %r9, %r26, %r6;
+; SM70-NEXT: sub.s32 %r10, %r25, %r5;
+; SM70-NEXT: selp.b32 %r11, %r10, %r25, %p2;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: selp.b32 %r12, %r9, %r26, %p1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p3 bra $L__BB219_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: $L__BB219_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd13, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: setp.ge.u32 %p4, %r28, %r8;
+; SM70-NEXT: setp.ge.u32 %p5, %r27, %r7;
+; SM70-NEXT: sub.s32 %r13, %r28, %r8;
+; SM70-NEXT: sub.s32 %r14, %r27, %r7;
+; SM70-NEXT: selp.b32 %r15, %r14, %r27, %p5;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NEXT: selp.b32 %r16, %r13, %r28, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
+; SM70-NEXT: setp.ne.b64 %p6, %rd21, %rd16;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p6 bra $L__BB219_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM70-NEXT: $L__BB219_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd23, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r30;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: setp.ge.u32 %p7, %r30, %r2;
+; SM70-NEXT: setp.ge.u32 %p8, %r29, %r1;
+; SM70-NEXT: sub.s32 %r17, %r30, %r2;
+; SM70-NEXT: sub.s32 %r18, %r29, %r1;
+; SM70-NEXT: selp.b32 %r19, %r18, %r29, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r19;
+; SM70-NEXT: selp.b32 %r20, %r17, %r30, %p7;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r20;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd26, %rd30;
+; SM70-NEXT: setp.ne.b64 %p9, %rd31, %rd26;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM70-NEXT: @%p9 bra $L__BB219_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM70-NEXT: $L__BB219_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: setp.ge.u32 %p10, %r32, %r4;
+; SM70-NEXT: setp.ge.u32 %p11, %r31, %r3;
+; SM70-NEXT: sub.s32 %r21, %r32, %r4;
+; SM70-NEXT: sub.s32 %r22, %r31, %r3;
+; SM70-NEXT: selp.b32 %r23, %r22, %r31, %p11;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: selp.b32 %r24, %r21, %r32, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd36, %rd40;
+; SM70-NEXT: setp.ne.b64 %p12, %rd41, %rd36;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM70-NEXT: @%p12 bra $L__BB219_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM70-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM70-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p2 bra $L__BB220_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p1, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd6, %rd10, %rd1;
+; SM70-NEXT: selp.b64 %rd7, %rd6, %rd10, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p2 bra $L__BB221_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB221_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p3, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd8, %rd11, %rd2;
+; SM70-NEXT: selp.b64 %rd9, %rd8, %rd11, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p4 bra $L__BB221_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: setp.ge.u64 %p1, %rd5, %rd3;
+; SM70-NEXT: sub.s64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.b64 %rd11, %rd10, %rd5, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd18, %rd5;
+; SM70-NEXT: @%p2 bra $L__BB222_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB222_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: setp.ge.u64 %p3, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd12, %rd6, %rd4;
+; SM70-NEXT: selp.b64 %rd13, %rd12, %rd6, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd19, %rd6;
+; SM70-NEXT: @%p4 bra $L__BB222_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB222_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p5, %rd20, %rd1;
+; SM70-NEXT: sub.s64 %rd14, %rd20, %rd1;
+; SM70-NEXT: selp.b64 %rd15, %rd14, %rd20, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p6, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p6 bra $L__BB222_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB222_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p7, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd16, %rd21, %rd2;
+; SM70-NEXT: selp.b64 %rd17, %rd16, %rd21, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB222_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: setp.ge.u64 %p1, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.b64 %rd19, %rd18, %rd7, %p1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd34, %rd7;
+; SM70-NEXT: @%p2 bra $L__BB223_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB223_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: setp.ge.u64 %p3, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd20, %rd8, %rd6;
+; SM70-NEXT: selp.b64 %rd21, %rd20, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd35, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB223_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB223_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: setp.ge.u64 %p5, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd22, %rd9, %rd3;
+; SM70-NEXT: selp.b64 %rd23, %rd22, %rd9, %p5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd36, %rd9;
+; SM70-NEXT: @%p6 bra $L__BB223_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB223_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: setp.ge.u64 %p7, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd24, %rd10, %rd4;
+; SM70-NEXT: selp.b64 %rd25, %rd24, %rd10, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd37, %rd10;
+; SM70-NEXT: @%p8 bra $L__BB223_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB223_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: setp.ge.u64 %p9, %rd13, %rd1;
+; SM70-NEXT: sub.s64 %rd26, %rd13, %rd1;
+; SM70-NEXT: selp.b64 %rd27, %rd26, %rd13, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd38, %rd13;
+; SM70-NEXT: @%p10 bra $L__BB223_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB223_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: setp.ge.u64 %p11, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd28, %rd14, %rd2;
+; SM70-NEXT: selp.b64 %rd29, %rd28, %rd14, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd39, %rd14;
+; SM70-NEXT: @%p12 bra $L__BB223_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB223_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p13, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd30, %rd40, %rd11;
+; SM70-NEXT: selp.b64 %rd31, %rd30, %rd40, %p13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p14, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p14 bra $L__BB223_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB223_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.ge.u64 %p15, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd32, %rd41, %rd12;
+; SM70-NEXT: selp.b64 %rd33, %rd32, %rd41, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p16 bra $L__BB223_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<6>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 255;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: and.b16 %rs3, %rs2, 255;
+; SM70-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB224_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r21, [%rd1];
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r21, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM70-NEXT: sub.s16 %rs9, %rs6, %rs8;
+; SM70-NEXT: setp.gt.u16 %p1, %rs9, %rs6;
+; SM70-NEXT: selp.b16 %rs10, -1, 0, %p1;
+; SM70-NEXT: sub.s16 %rs11, %rs5, %rs7;
+; SM70-NEXT: setp.gt.u16 %p2, %rs11, %rs5;
+; SM70-NEXT: selp.b16 %rs12, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r12, {%rs12, %rs10};
+; SM70-NEXT: xor.b32 %r13, %r12, -1;
+; SM70-NEXT: mov.b32 %r14, {%rs11, %rs9};
+; SM70-NEXT: and.b32 %r15, %r14, %r13;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r15;
+; SM70-NEXT: shl.b16 %rs15, %rs14, 8;
+; SM70-NEXT: or.b16 %rs16, %rs13, %rs15;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, %r1;
+; SM70-NEXT: and.b32 %r18, %r21, %r2;
+; SM70-NEXT: or.b32 %r19, %r18, %r17;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r21, %r19;
+; SM70-NEXT: setp.ne.b32 %p3, %r3, %r21;
+; SM70-NEXT: mov.b32 %r21, %r3;
+; SM70-NEXT: @%p3 bra $L__BB225_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r20, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM70-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM70-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM70-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB226_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM70-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM70-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM70-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM70-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM70-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM70-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM70-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM70-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM70-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM70-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM70-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM70-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM70-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM70-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB227_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB228_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r7;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: and.b32 %r6, %r3, %r5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r7;
+; SM70-NEXT: mov.b32 %r7, %r1;
+; SM70-NEXT: @%p3 bra $L__BB229_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r11;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r4, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: and.b32 %r6, %r3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r12;
+; SM70-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM70-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM70-NEXT: mov.b32 %r7, {%rs14, %rs13};
+; SM70-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM70-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM70-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM70-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM70-NEXT: mov.b32 %r8, {%rs16, %rs15};
+; SM70-NEXT: xor.b32 %r9, %r8, -1;
+; SM70-NEXT: and.b32 %r10, %r7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB230_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<33>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM70-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM70-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM70-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM70-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM70-NEXT: setp.gt.u16 %p1, %rs5, %rs4;
+; SM70-NEXT: selp.b16 %rs7, -1, 0, %p1;
+; SM70-NEXT: setp.gt.u16 %p2, %rs6, %rs3;
+; SM70-NEXT: selp.b16 %rs8, -1, 0, %p2;
+; SM70-NEXT: mov.b32 %r6, {%rs8, %rs7};
+; SM70-NEXT: xor.b32 %r7, %r6, -1;
+; SM70-NEXT: and.b32 %r8, %r5, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r22;
+; SM70-NEXT: sub.s16 %rs13, %rs12, %rs10;
+; SM70-NEXT: sub.s16 %rs14, %rs11, %rs9;
+; SM70-NEXT: mov.b32 %r9, {%rs14, %rs13};
+; SM70-NEXT: setp.gt.u16 %p3, %rs13, %rs12;
+; SM70-NEXT: selp.b16 %rs15, -1, 0, %p3;
+; SM70-NEXT: setp.gt.u16 %p4, %rs14, %rs11;
+; SM70-NEXT: selp.b16 %rs16, -1, 0, %p4;
+; SM70-NEXT: mov.b32 %r10, {%rs16, %rs15};
+; SM70-NEXT: xor.b32 %r11, %r10, -1;
+; SM70-NEXT: and.b32 %r12, %r9, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB231_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd12;
+; SM70-NEXT: $L__BB231_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r3;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r23;
+; SM70-NEXT: sub.s16 %rs21, %rs20, %rs18;
+; SM70-NEXT: sub.s16 %rs22, %rs19, %rs17;
+; SM70-NEXT: mov.b32 %r13, {%rs22, %rs21};
+; SM70-NEXT: setp.gt.u16 %p6, %rs21, %rs20;
+; SM70-NEXT: selp.b16 %rs23, -1, 0, %p6;
+; SM70-NEXT: setp.gt.u16 %p7, %rs22, %rs19;
+; SM70-NEXT: selp.b16 %rs24, -1, 0, %p7;
+; SM70-NEXT: mov.b32 %r14, {%rs24, %rs23};
+; SM70-NEXT: xor.b32 %r15, %r14, -1;
+; SM70-NEXT: and.b32 %r16, %r13, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r4;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r24;
+; SM70-NEXT: sub.s16 %rs29, %rs28, %rs26;
+; SM70-NEXT: sub.s16 %rs30, %rs27, %rs25;
+; SM70-NEXT: mov.b32 %r17, {%rs30, %rs29};
+; SM70-NEXT: setp.gt.u16 %p8, %rs29, %rs28;
+; SM70-NEXT: selp.b16 %rs31, -1, 0, %p8;
+; SM70-NEXT: setp.gt.u16 %p9, %rs30, %rs27;
+; SM70-NEXT: selp.b16 %rs32, -1, 0, %p9;
+; SM70-NEXT: mov.b32 %r18, {%rs32, %rs31};
+; SM70-NEXT: xor.b32 %r19, %r18, -1;
+; SM70-NEXT: and.b32 %r20, %r17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r20;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r24;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB231_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r3, %r5, %r1;
+; SM70-NEXT: sub.s32 %r4, %r3, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB232_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r3, %r7, %r1;
+; SM70-NEXT: sub.s32 %r4, %r3, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM70-NEXT: max.u32 %r5, %r8, %r2;
+; SM70-NEXT: sub.s32 %r6, %r5, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB233_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r5, %r13, %r1;
+; SM70-NEXT: sub.s32 %r6, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: max.u32 %r7, %r14, %r2;
+; SM70-NEXT: sub.s32 %r8, %r7, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB234_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd12;
+; SM70-NEXT: $L__BB234_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r9, %r15, %r3;
+; SM70-NEXT: sub.s32 %r10, %r9, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM70-NEXT: max.u32 %r11, %r16, %r4;
+; SM70-NEXT: sub.s32 %r12, %r11, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r15, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB234_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<33>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r9, %r25, %r5;
+; SM70-NEXT: sub.s32 %r10, %r9, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r10;
+; SM70-NEXT: max.u32 %r11, %r26, %r6;
+; SM70-NEXT: sub.s32 %r12, %r11, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB235_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: $L__BB235_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r13, %r27, %r7;
+; SM70-NEXT: sub.s32 %r14, %r13, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r14;
+; SM70-NEXT: max.u32 %r15, %r28, %r8;
+; SM70-NEXT: sub.s32 %r16, %r15, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r16;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB235_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd22;
+; SM70-NEXT: $L__BB235_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r17, %r29, %r1;
+; SM70-NEXT: sub.s32 %r18, %r17, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r18;
+; SM70-NEXT: max.u32 %r19, %r30, %r2;
+; SM70-NEXT: sub.s32 %r20, %r19, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r20;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r30;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB235_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd32;
+; SM70-NEXT: $L__BB235_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u32 %r21, %r31, %r3;
+; SM70-NEXT: sub.s32 %r22, %r21, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM70-NEXT: max.u32 %r23, %r32, %r4;
+; SM70-NEXT: sub.s32 %r24, %r23, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r31;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r32;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r31, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB235_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<7>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM70-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM70-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM70-NEXT: mov.b64 %rd6, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB236_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd10, [%rd5];
+; SM70-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd6, %rd10, %rd1;
+; SM70-NEXT: sub.s64 %rd7, %rd6, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd10, %rd7;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd10;
+; SM70-NEXT: mov.b64 %rd10, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB237_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
+; SM70-NEXT: $L__BB237_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd8, %rd11, %rd2;
+; SM70-NEXT: sub.s64 %rd9, %rd8, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd11;
+; SM70-NEXT: mov.b64 %rd11, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB237_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd18, [%rd9];
+; SM70-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd18;
+; SM70-NEXT: max.u64 %rd10, %rd5, %rd3;
+; SM70-NEXT: sub.s64 %rd11, %rd10, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd5, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd18, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB238_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
+; SM70-NEXT: $L__BB238_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd19;
+; SM70-NEXT: max.u64 %rd12, %rd6, %rd4;
+; SM70-NEXT: sub.s64 %rd13, %rd12, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd6, %rd13;
+; SM70-NEXT: setp.ne.b64 %p2, %rd19, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB238_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
+; SM70-NEXT: $L__BB238_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd14, %rd20, %rd1;
+; SM70-NEXT: sub.s64 %rd15, %rd14, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd20, %rd15;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd20;
+; SM70-NEXT: mov.b64 %rd20, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB238_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
+; SM70-NEXT: $L__BB238_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd16, %rd21, %rd2;
+; SM70-NEXT: sub.s64 %rd17, %rd16, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd21;
+; SM70-NEXT: mov.b64 %rd21, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB238_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd18, %rd19};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM70-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd34, [%rd17];
+; SM70-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd34;
+; SM70-NEXT: max.u64 %rd18, %rd7, %rd5;
+; SM70-NEXT: sub.s64 %rd19, %rd18, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd7, %rd19;
+; SM70-NEXT: setp.ne.b64 %p1, %rd34, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB239_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
+; SM70-NEXT: $L__BB239_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd35;
+; SM70-NEXT: max.u64 %rd20, %rd8, %rd6;
+; SM70-NEXT: sub.s64 %rd21, %rd20, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd8, %rd21;
+; SM70-NEXT: setp.ne.b64 %p2, %rd35, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB239_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
+; SM70-NEXT: $L__BB239_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd36;
+; SM70-NEXT: max.u64 %rd22, %rd9, %rd3;
+; SM70-NEXT: sub.s64 %rd23, %rd22, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd9, %rd23;
+; SM70-NEXT: setp.ne.b64 %p3, %rd36, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB239_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
+; SM70-NEXT: $L__BB239_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd37;
+; SM70-NEXT: max.u64 %rd24, %rd10, %rd4;
+; SM70-NEXT: sub.s64 %rd25, %rd24, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd10, %rd25;
+; SM70-NEXT: setp.ne.b64 %p4, %rd37, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB239_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
+; SM70-NEXT: $L__BB239_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd38;
+; SM70-NEXT: max.u64 %rd26, %rd13, %rd1;
+; SM70-NEXT: sub.s64 %rd27, %rd26, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd13, %rd27;
+; SM70-NEXT: setp.ne.b64 %p5, %rd38, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB239_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
+; SM70-NEXT: $L__BB239_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd39;
+; SM70-NEXT: max.u64 %rd28, %rd14, %rd2;
+; SM70-NEXT: sub.s64 %rd29, %rd28, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd14, %rd29;
+; SM70-NEXT: setp.ne.b64 %p6, %rd39, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB239_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
+; SM70-NEXT: $L__BB239_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd30, %rd40, %rd11;
+; SM70-NEXT: sub.s64 %rd31, %rd30, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd40, %rd31;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd40;
+; SM70-NEXT: mov.b64 %rd40, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB239_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
+; SM70-NEXT: $L__BB239_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.u64 %rd32, %rd41, %rd12;
+; SM70-NEXT: sub.s64 %rd33, %rd32, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd41;
+; SM70-NEXT: mov.b64 %rd41, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB239_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd34, %rd35};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB244_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB245_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB246_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB246_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB246_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: sub.rn.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB247_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB247_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: sub.rn.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB247_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB247_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: sub.rn.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB247_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB247_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: sub.rn.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB247_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB248_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: min.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB249_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: min.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB250_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB250_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: min.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB250_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: min.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB251_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB251_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: min.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB251_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB251_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: min.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB251_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB251_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: min.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB251_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r3, %r4, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM70-NEXT: mov.b32 %r4, %r2;
+; SM70-NEXT: @%p1 bra $L__BB252_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: max.f32 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB253_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: max.f32 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB254_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB254_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: max.f32 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB254_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r9, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: max.f32 %r10, %r18, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB255_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd12;
+; SM70-NEXT: $L__BB255_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r11, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r11;
+; SM70-NEXT: max.f32 %r12, %r20, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r12;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB255_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd22;
+; SM70-NEXT: $L__BB255_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r13, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r13;
+; SM70-NEXT: max.f32 %r14, %r22, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r14;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r22;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p3, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd31;
+; SM70-NEXT: @%p3 bra $L__BB255_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd32;
+; SM70-NEXT: $L__BB255_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f32 %r15, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM70-NEXT: max.f32 %r16, %r24, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r24;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p4, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd41;
+; SM70-NEXT: @%p4 bra $L__BB255_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM70-NEXT: min.f32 %r3, %r8, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM70-NEXT: mov.b32 %r8, %r2;
+; SM70-NEXT: @%p5 bra $L__BB256_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM70-NEXT: min.f32 %r3, %r13, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
+; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM70-NEXT: min.f32 %r8, %r14, %r2;
+; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
+; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB257_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM70-NEXT: min.f32 %r5, %r25, %r1;
+; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
+; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM70-NEXT: min.f32 %r10, %r26, %r2;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
+; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB258_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM70-NEXT: $L__BB258_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM70-NEXT: min.f32 %r15, %r27, %r3;
+; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
+; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM70-NEXT: min.f32 %r20, %r28, %r4;
+; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
+; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB258_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<37>;
+; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM70-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM70-NEXT: min.f32 %r9, %r49, %r5;
+; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
+; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM70-NEXT: min.f32 %r14, %r50, %r6;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
+; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB259_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM70-NEXT: $L__BB259_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM70-NEXT: min.f32 %r19, %r51, %r7;
+; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
+; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM70-NEXT: min.f32 %r24, %r52, %r8;
+; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
+; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB259_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM70-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM70-NEXT: $L__BB259_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM70-NEXT: min.f32 %r29, %r53, %r1;
+; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM70-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
+; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM70-NEXT: min.f32 %r34, %r54, %r2;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
+; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM70-NEXT: @%p27 bra $L__BB259_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM70-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
+; SM70-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM70-NEXT: $L__BB259_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM70-NEXT: min.f32 %r39, %r55, %r3;
+; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM70-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
+; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM70-NEXT: min.f32 %r44, %r56, %r4;
+; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM70-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
+; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM70-NEXT: @%p36 bra $L__BB259_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
+; SM70-NEXT: max.f32 %r3, %r8, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
+; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
+; SM70-NEXT: mov.b32 %r8, %r2;
+; SM70-NEXT: @%p5 bra $L__BB260_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
+; SM70-NEXT: max.f32 %r3, %r13, %r1;
+; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r13, 0;
+; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
+; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
+; SM70-NEXT: max.f32 %r8, %r14, %r2;
+; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r14, 0;
+; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
+; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
+; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB261_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<19>;
+; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
+; SM70-NEXT: max.f32 %r5, %r25, %r1;
+; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r25, 0;
+; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
+; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
+; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
+; SM70-NEXT: max.f32 %r10, %r26, %r2;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r26, 0;
+; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
+; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
+; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB262_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM70-NEXT: $L__BB262_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
+; SM70-NEXT: max.f32 %r15, %r27, %r3;
+; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r27, 0;
+; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
+; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
+; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
+; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
+; SM70-NEXT: max.f32 %r20, %r28, %r4;
+; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r28, 0;
+; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
+; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
+; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB262_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<37>;
+; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .b64 %rd<42>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM70-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM70-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
+; SM70-NEXT: max.f32 %r9, %r49, %r5;
+; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
+; SM70-NEXT: setp.eq.b32 %p2, %r49, 0;
+; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
+; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
+; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
+; SM70-NEXT: max.f32 %r14, %r50, %r6;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
+; SM70-NEXT: setp.eq.b32 %p6, %r50, 0;
+; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
+; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
+; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
+; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM70-NEXT: @%p9 bra $L__BB263_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
+; SM70-NEXT: setp.eq.b32 %p12, %r7, 0;
+; SM70-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM70-NEXT: $L__BB263_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
+; SM70-NEXT: max.f32 %r19, %r51, %r7;
+; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
+; SM70-NEXT: setp.eq.b32 %p11, %r51, 0;
+; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
+; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
+; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
+; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
+; SM70-NEXT: max.f32 %r24, %r52, %r8;
+; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
+; SM70-NEXT: setp.eq.b32 %p15, %r52, 0;
+; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
+; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
+; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
+; SM70-NEXT: @%p18 bra $L__BB263_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
+; SM70-NEXT: setp.eq.b32 %p21, %r1, 0;
+; SM70-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM70-NEXT: $L__BB263_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
+; SM70-NEXT: max.f32 %r29, %r53, %r1;
+; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
+; SM70-NEXT: setp.eq.b32 %p20, %r53, 0;
+; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
+; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
+; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
+; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
+; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
+; SM70-NEXT: max.f32 %r34, %r54, %r2;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.b32 %p24, %r54, 0;
+; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
+; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
+; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
+; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
+; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
+; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
+; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
+; SM70-NEXT: @%p27 bra $L__BB263_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
+; SM70-NEXT: setp.eq.b32 %p30, %r3, 0;
+; SM70-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM70-NEXT: $L__BB263_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
+; SM70-NEXT: max.f32 %r39, %r55, %r3;
+; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
+; SM70-NEXT: setp.eq.b32 %p29, %r55, 0;
+; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
+; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
+; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
+; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
+; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
+; SM70-NEXT: max.f32 %r44, %r56, %r4;
+; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
+; SM70-NEXT: setp.eq.b32 %p33, %r56, 0;
+; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
+; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
+; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
+; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
+; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
+; SM70-NEXT: @%p36 bra $L__BB263_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB268_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB269_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB269_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB269_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB269_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB270_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: sub.rn.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB270_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB270_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: sub.rn.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB270_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB270_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB270_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB270_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB270_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB271_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: sub.rn.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB271_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB271_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: sub.rn.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB271_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB271_5: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: sub.rn.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB271_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end19
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB271_7: // %atomicrmw.start26
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: sub.rn.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB271_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end25
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB271_9: // %atomicrmw.start41
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: sub.rn.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB271_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end40
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB271_11: // %atomicrmw.start47
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: sub.rn.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB271_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end46
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB271_13: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB271_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end57
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB271_15: // %atomicrmw.start64
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB271_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end63
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB272_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB273_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB273_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB273_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB273_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB274_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB274_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB274_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: min.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB274_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB274_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB274_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB274_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB274_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB275_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB275_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB275_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: min.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB275_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB275_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: min.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB275_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB275_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: min.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB275_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB275_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: min.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB275_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB275_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: min.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB275_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB275_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB275_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB275_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: min.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB275_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM70-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM70-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM70-NEXT: mov.b64 %rd5, %rd2;
+; SM70-NEXT: @%p1 bra $L__BB276_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];
+; SM70-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd6, %rd8, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd8, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd3, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd3;
+; SM70-NEXT: @%p1 bra $L__BB277_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
+; SM70-NEXT: $L__BB277_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd7, %rd9, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
+; SM70-NEXT: setp.ne.b64 %p2, %rd4, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd4;
+; SM70-NEXT: @%p2 bra $L__BB277_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd9];
+; SM70-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd14;
+; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd5, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd14, %rd5;
+; SM70-NEXT: @%p1 bra $L__BB278_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
+; SM70-NEXT: $L__BB278_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd15;
+; SM70-NEXT: max.f64 %rd11, %rd6, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p2, %rd15, %rd6;
+; SM70-NEXT: @%p2 bra $L__BB278_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
+; SM70-NEXT: $L__BB278_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd12, %rd16, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd16, %rd12;
+; SM70-NEXT: setp.ne.b64 %p3, %rd7, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd7;
+; SM70-NEXT: @%p3 bra $L__BB278_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
+; SM70-NEXT: $L__BB278_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd13, %rd17, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd8, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd8;
+; SM70-NEXT: @%p4 bra $L__BB278_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd14, %rd15};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd17];
+; SM70-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd26;
+; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd7, %rd18;
+; SM70-NEXT: setp.ne.b64 %p1, %rd26, %rd7;
+; SM70-NEXT: @%p1 bra $L__BB279_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
+; SM70-NEXT: $L__BB279_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd27;
+; SM70-NEXT: max.f64 %rd19, %rd8, %rd6;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd8, %rd19;
+; SM70-NEXT: setp.ne.b64 %p2, %rd27, %rd8;
+; SM70-NEXT: @%p2 bra $L__BB279_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
+; SM70-NEXT: $L__BB279_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd28;
+; SM70-NEXT: max.f64 %rd20, %rd9, %rd3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd9, %rd20;
+; SM70-NEXT: setp.ne.b64 %p3, %rd28, %rd9;
+; SM70-NEXT: @%p3 bra $L__BB279_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
+; SM70-NEXT: $L__BB279_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd29;
+; SM70-NEXT: max.f64 %rd21, %rd10, %rd4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd10, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd29, %rd10;
+; SM70-NEXT: @%p4 bra $L__BB279_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
+; SM70-NEXT: $L__BB279_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd30;
+; SM70-NEXT: max.f64 %rd22, %rd13, %rd1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd13, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd13;
+; SM70-NEXT: @%p5 bra $L__BB279_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
+; SM70-NEXT: $L__BB279_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd31;
+; SM70-NEXT: max.f64 %rd23, %rd14, %rd2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd14, %rd23;
+; SM70-NEXT: setp.ne.b64 %p6, %rd31, %rd14;
+; SM70-NEXT: @%p6 bra $L__BB279_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
+; SM70-NEXT: $L__BB279_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd24, %rd32, %rd11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p7, %rd15, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd15;
+; SM70-NEXT: @%p7 bra $L__BB279_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
+; SM70-NEXT: $L__BB279_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: max.f64 %rd25, %rd33, %rd12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd16, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd16;
+; SM70-NEXT: @%p8 bra $L__BB279_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM70-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd2;
+; SM70-NEXT: @%p5 bra $L__BB280_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM70-NEXT: min.f64 %rd6, %rd16, %rd1;
+; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd3;
+; SM70-NEXT: @%p5 bra $L__BB281_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB281_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM70-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd4;
+; SM70-NEXT: @%p10 bra $L__BB281_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<21>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
+; SM70-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd30;
+; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM70-NEXT: @%p5 bra $L__BB282_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM70-NEXT: $L__BB282_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: min.f64 %rd15, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM70-NEXT: @%p10 bra $L__BB282_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB282_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM70-NEXT: min.f64 %rd20, %rd32, %rd1;
+; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd7;
+; SM70-NEXT: @%p15 bra $L__BB282_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB282_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM70-NEXT: min.f64 %rd25, %rd33, %rd2;
+; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd8;
+; SM70-NEXT: @%p20 bra $L__BB282_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<41>;
+; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
+; SM70-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd58;
+; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM70-NEXT: @%p5 bra $L__BB283_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM70-NEXT: $L__BB283_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: min.f64 %rd23, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM70-NEXT: @%p10 bra $L__BB283_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM70-NEXT: $L__BB283_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd60;
+; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: min.f64 %rd28, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM70-NEXT: @%p15 bra $L__BB283_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM70-NEXT: $L__BB283_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd61;
+; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM70-NEXT: min.f64 %rd33, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM70-NEXT: @%p20 bra $L__BB283_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
+; SM70-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM70-NEXT: $L__BB283_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd62;
+; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM70-NEXT: min.f64 %rd38, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM70-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM70-NEXT: @%p25 bra $L__BB283_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
+; SM70-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM70-NEXT: $L__BB283_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd63;
+; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM70-NEXT: min.f64 %rd43, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM70-NEXT: @%p30 bra $L__BB283_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
+; SM70-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM70-NEXT: $L__BB283_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM70-NEXT: min.f64 %rd48, %rd64, %rd11;
+; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM70-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM70-NEXT: mov.b64 %rd64, %rd15;
+; SM70-NEXT: @%p35 bra $L__BB283_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
+; SM70-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM70-NEXT: $L__BB283_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM70-NEXT: min.f64 %rd53, %rd65, %rd12;
+; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM70-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM70-NEXT: mov.b64 %rd65, %rd16;
+; SM70-NEXT: @%p40 bra $L__BB283_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM70-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM70-NEXT: mov.b64 %rd9, %rd2;
+; SM70-NEXT: @%p5 bra $L__BB284_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
+; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
+; SM70-NEXT: max.f64 %rd6, %rd16, %rd1;
+; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
+; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
+; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
+; SM70-NEXT: mov.b64 %rd16, %rd3;
+; SM70-NEXT: @%p5 bra $L__BB285_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM70-NEXT: $L__BB285_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
+; SM70-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd17, 0;
+; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
+; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
+; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
+; SM70-NEXT: mov.b64 %rd17, %rd4;
+; SM70-NEXT: @%p10 bra $L__BB285_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<21>;
+; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
+; SM70-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd5, %rd30;
+; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
+; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd5, 0;
+; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
+; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
+; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
+; SM70-NEXT: @%p5 bra $L__BB286_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM70-NEXT: $L__BB286_3: // %atomicrmw.start5
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
+; SM70-NEXT: max.f64 %rd15, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
+; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
+; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
+; SM70-NEXT: @%p10 bra $L__BB286_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end4
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM70-NEXT: $L__BB286_5: // %atomicrmw.start15
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
+; SM70-NEXT: max.f64 %rd20, %rd32, %rd1;
+; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd32, 0;
+; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
+; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
+; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
+; SM70-NEXT: mov.b64 %rd32, %rd7;
+; SM70-NEXT: @%p15 bra $L__BB286_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end14
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM70-NEXT: $L__BB286_7: // %atomicrmw.start20
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
+; SM70-NEXT: max.f64 %rd25, %rd33, %rd2;
+; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd33, 0;
+; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
+; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
+; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
+; SM70-NEXT: mov.b64 %rd33, %rd8;
+; SM70-NEXT: @%p20 bra $L__BB286_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end19
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<41>;
+; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b64 {%rd11, %rd12}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
+; SM70-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd7, %rd58;
+; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
+; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
+; SM70-NEXT: setp.eq.b64 %p2, %rd7, 0;
+; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
+; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
+; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
+; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
+; SM70-NEXT: @%p5 bra $L__BB287_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
+; SM70-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM70-NEXT: $L__BB287_3: // %atomicrmw.start9
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
+; SM70-NEXT: max.f64 %rd23, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
+; SM70-NEXT: setp.eq.b64 %p7, %rd8, 0;
+; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
+; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
+; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
+; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
+; SM70-NEXT: @%p10 bra $L__BB287_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end8
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
+; SM70-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM70-NEXT: $L__BB287_5: // %atomicrmw.start19
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd9, %rd60;
+; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: max.f64 %rd28, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
+; SM70-NEXT: setp.eq.b64 %p12, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
+; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
+; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
+; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
+; SM70-NEXT: @%p15 bra $L__BB287_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end18
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
+; SM70-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM70-NEXT: $L__BB287_7: // %atomicrmw.start24
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd10, %rd61;
+; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
+; SM70-NEXT: max.f64 %rd33, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
+; SM70-NEXT: setp.eq.b64 %p17, %rd10, 0;
+; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
+; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
+; SM70-NEXT: @%p20 bra $L__BB287_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end23
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
+; SM70-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM70-NEXT: $L__BB287_9: // %atomicrmw.start38
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd13, %rd62;
+; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
+; SM70-NEXT: max.f64 %rd38, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
+; SM70-NEXT: setp.eq.b64 %p22, %rd13, 0;
+; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
+; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
+; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
+; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
+; SM70-NEXT: @%p25 bra $L__BB287_9;
+; SM70-NEXT: // %bb.10: // %atomicrmw.end37
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
+; SM70-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM70-NEXT: $L__BB287_11: // %atomicrmw.start43
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b64 %rd14, %rd63;
+; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
+; SM70-NEXT: max.f64 %rd43, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.b64 %p27, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
+; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
+; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
+; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
+; SM70-NEXT: @%p30 bra $L__BB287_11;
+; SM70-NEXT: // %bb.12: // %atomicrmw.end42
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
+; SM70-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM70-NEXT: $L__BB287_13: // %atomicrmw.start53
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
+; SM70-NEXT: max.f64 %rd48, %rd64, %rd11;
+; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
+; SM70-NEXT: setp.eq.b64 %p32, %rd64, 0;
+; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
+; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
+; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
+; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
+; SM70-NEXT: mov.b64 %rd64, %rd15;
+; SM70-NEXT: @%p35 bra $L__BB287_13;
+; SM70-NEXT: // %bb.14: // %atomicrmw.end52
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
+; SM70-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM70-NEXT: $L__BB287_15: // %atomicrmw.start58
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
+; SM70-NEXT: max.f64 %rd53, %rd65, %rd12;
+; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
+; SM70-NEXT: setp.eq.b64 %p37, %rd65, 0;
+; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
+; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
+; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
+; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
+; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
+; SM70-NEXT: mov.b64 %rd65, %rd16;
+; SM70-NEXT: @%p40 bra $L__BB287_15;
+; SM70-NEXT: // %bb.16: // %atomicrmw.end57
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
+; SM70-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM70-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM70-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM70-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p1 bra $L__BB292_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p1 bra $L__BB293_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB294_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB295_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB295_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB295_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NEXT: min.f32 %r11, %r9, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB296_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<11>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r10, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: min.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: min.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM70-NEXT: mov.b32 %r10, %r1;
+; SM70-NEXT: @%p1 bra $L__BB297_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM70-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: min.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: min.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM70-NEXT: min.f32 %r12, %r11, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM70-NEXT: min.f32 %r15, %r14, %r13;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM70-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB298_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<37>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM70-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM70-NEXT: min.f32 %r7, %r6, %r5;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM70-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM70-NEXT: min.f32 %r10, %r9, %r8;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM70-NEXT: min.f32 %r14, %r13, %r12;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM70-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM70-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM70-NEXT: min.f32 %r17, %r16, %r15;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM70-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB299_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM70-NEXT: $L__BB299_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM70-NEXT: min.f32 %r21, %r20, %r19;
+; SM70-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM70-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM70-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM70-NEXT: min.f32 %r24, %r23, %r22;
+; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM70-NEXT: min.f32 %r28, %r27, %r26;
+; SM70-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM70-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM70-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM70-NEXT: min.f32 %r31, %r30, %r29;
+; SM70-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM70-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB299_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<4>;
+; SM70-NEXT: .reg .b32 %r<18>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NEXT: max.f32 %r11, %r9, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p1 bra $L__BB300_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<11>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r10, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: $L__BB301_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r10;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: max.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: max.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r10;
+; SM70-NEXT: mov.b32 %r10, %r1;
+; SM70-NEXT: @%p1 bra $L__BB301_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
+; SM70-NEXT: $L__BB302_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
+; SM70-NEXT: max.f32 %r5, %r4, %r3;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
+; SM70-NEXT: max.f32 %r8, %r7, %r6;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
+; SM70-NEXT: mov.b32 %r9, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r18;
+; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
+; SM70-NEXT: max.f32 %r12, %r11, %r10;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
+; SM70-NEXT: max.f32 %r15, %r14, %r13;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
+; SM70-NEXT: mov.b32 %r16, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r18;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB302_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r17, %r18};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<37>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
+; SM70-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
+; SM70-NEXT: max.f32 %r7, %r6, %r5;
+; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r7;
+; SM70-NEXT: cvt.f32.f16 %r8, %rs1;
+; SM70-NEXT: cvt.f32.f16 %r9, %rs3;
+; SM70-NEXT: max.f32 %r10, %r9, %r8;
+; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
+; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
+; SM70-NEXT: max.f32 %r14, %r13, %r12;
+; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r14;
+; SM70-NEXT: cvt.f32.f16 %r15, %rs7;
+; SM70-NEXT: cvt.f32.f16 %r16, %rs9;
+; SM70-NEXT: max.f32 %r17, %r16, %r15;
+; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r17;
+; SM70-NEXT: mov.b32 %r18, {%rs12, %rs11};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB303_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
+; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
+; SM70-NEXT: $L__BB303_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
+; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
+; SM70-NEXT: max.f32 %r21, %r20, %r19;
+; SM70-NEXT: cvt.rn.f16.f32 %rs17, %r21;
+; SM70-NEXT: cvt.f32.f16 %r22, %rs13;
+; SM70-NEXT: cvt.f32.f16 %r23, %rs15;
+; SM70-NEXT: max.f32 %r24, %r23, %r22;
+; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
+; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
+; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
+; SM70-NEXT: max.f32 %r28, %r27, %r26;
+; SM70-NEXT: cvt.rn.f16.f32 %rs23, %r28;
+; SM70-NEXT: cvt.f32.f16 %r29, %rs19;
+; SM70-NEXT: cvt.f32.f16 %r30, %rs21;
+; SM70-NEXT: max.f32 %r31, %r30, %r29;
+; SM70-NEXT: cvt.rn.f16.f32 %rs24, %r31;
+; SM70-NEXT: mov.b32 %r32, {%rs24, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r32;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r35;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r36;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p2, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r35, %rd21;
+; SM70-NEXT: @%p2 bra $L__BB303_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p6 bra $L__BB304_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: $L__BB305_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p11 bra $L__BB305_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: $L__BB306_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB306_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: $L__BB307_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB307_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB307_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM70-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs33, -32768;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs31, -32768;
+; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM70-NEXT: mov.b16 %rs38, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM70-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs32, -32768;
+; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs30, -32768;
+; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM70-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs48, -32768;
+; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs46, -32768;
+; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM70-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs47, -32768;
+; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs45, -32768;
+; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB307_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: mov.b32 %r14, %r3;
+; SM70-NEXT: @%p6 bra $L__BB308_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: $L__BB309_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: mov.b32 %r4, %r1;
+; SM70-NEXT: @%p11 bra $L__BB309_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: $L__BB310_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
+; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB310_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
+; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: $L__BB311_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: mov.b16 %rs9, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
+; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
+; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
+; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
+; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
+; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
+; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
+; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
+; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
+; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
+; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
+; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
+; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
+; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
+; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
+; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
+; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
+; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
+; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r10;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB311_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
+; SM70-NEXT: $L__BB311_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
+; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
+; SM70-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
+; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs33, 0;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs31, 0;
+; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
+; SM70-NEXT: mov.b16 %rs38, 0x0000;
+; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
+; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
+; SM70-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
+; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs32, 0;
+; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs30, 0;
+; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
+; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
+; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
+; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
+; SM70-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
+; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
+; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs48, 0;
+; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs46, 0;
+; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
+; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
+; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
+; SM70-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
+; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
+; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs47, 0;
+; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs45, 0;
+; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
+; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
+; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
+; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB311_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB312_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB312_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB313_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB313_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB314_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: add.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: add.rn.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: add.rn.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB314_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB315_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: add.rn.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: add.rn.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: add.rn.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: add.rn.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB315_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB315_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: add.rn.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: add.rn.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: add.rn.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: add.rn.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB315_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB316_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB317_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB317_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB318_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: sub.rn.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: sub.rn.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: sub.rn.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB318_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB319_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: sub.rn.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: sub.rn.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: sub.rn.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: sub.rn.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB319_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB319_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: sub.rn.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: sub.rn.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: sub.rn.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: sub.rn.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB319_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: min.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB320_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB321_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: min.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: min.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB321_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB322_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: min.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: min.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: min.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: min.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB322_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB323_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: min.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: min.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: min.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: min.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB323_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB323_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: min.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: min.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: min.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: min.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB323_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b16 %rs<2>;
+; SM70-NEXT: .reg .b32 %r<24>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: max.f32 %r12, %r9, %r11;
+; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NEXT: mov.b32 %r23, %r3;
+; SM70-NEXT: @%p2 bra $L__BB324_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<4>;
+; SM70-NEXT: .reg .b16 %rs<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: $L__BB325_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r24;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: max.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: max.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r24, %r23;
+; SM70-NEXT: setp.ne.b32 %p3, %r1, %r24;
+; SM70-NEXT: mov.b32 %r24, %r1;
+; SM70-NEXT: @%p3 bra $L__BB325_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<47>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: $L__BB326_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: max.f32 %r7, %r6, %r4;
+; SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT: add.s32 %r9, %r8, %r7;
+; SM70-NEXT: add.s32 %r10, %r9, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
+; SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: max.f32 %r17, %r16, %r14;
+; SM70-NEXT: bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r17;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r17, %r17;
+; SM70-NEXT: or.b32 %r21, %r17, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r12, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r23;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r46;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs8;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: max.f32 %r28, %r27, %r25;
+; SM70-NEXT: bfe.u32 %r29, %r28, 16, 1;
+; SM70-NEXT: add.s32 %r30, %r29, %r28;
+; SM70-NEXT: add.s32 %r31, %r30, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; SM70-NEXT: or.b32 %r32, %r28, 4194304;
+; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: max.f32 %r38, %r37, %r35;
+; SM70-NEXT: bfe.u32 %r39, %r38, 16, 1;
+; SM70-NEXT: add.s32 %r40, %r39, %r38;
+; SM70-NEXT: add.s32 %r41, %r40, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r38, %r38;
+; SM70-NEXT: or.b32 %r42, %r38, 4194304;
+; SM70-NEXT: selp.b32 %r43, %r42, %r41, %p4;
+; SM70-NEXT: prmt.b32 %r44, %r43, %r33, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r44;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r46;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB326_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r45, %r46};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<11>;
+; SM70-NEXT: .reg .b16 %rs<17>;
+; SM70-NEXT: .reg .b32 %r<93>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: $L__BB327_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: max.f32 %r9, %r8, %r6;
+; SM70-NEXT: bfe.u32 %r10, %r9, 16, 1;
+; SM70-NEXT: add.s32 %r11, %r10, %r9;
+; SM70-NEXT: add.s32 %r12, %r11, 32767;
+; SM70-NEXT: setp.nan.f32 %p1, %r9, %r9;
+; SM70-NEXT: or.b32 %r13, %r9, 4194304;
+; SM70-NEXT: selp.b32 %r14, %r13, %r12, %p1;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs1;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: cvt.u32.u16 %r17, %rs3;
+; SM70-NEXT: shl.b32 %r18, %r17, 16;
+; SM70-NEXT: max.f32 %r19, %r18, %r16;
+; SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
+; SM70-NEXT: add.s32 %r21, %r20, %r19;
+; SM70-NEXT: add.s32 %r22, %r21, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
+; SM70-NEXT: or.b32 %r23, %r19, 4194304;
+; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: max.f32 %r30, %r29, %r27;
+; SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; SM70-NEXT: add.s32 %r32, %r31, %r30;
+; SM70-NEXT: add.s32 %r33, %r32, 32767;
+; SM70-NEXT: setp.nan.f32 %p3, %r30, %r30;
+; SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; SM70-NEXT: selp.b32 %r35, %r34, %r33, %p3;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs5;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs7;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: max.f32 %r40, %r39, %r37;
+; SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
+; SM70-NEXT: add.s32 %r42, %r41, %r40;
+; SM70-NEXT: add.s32 %r43, %r42, 32767;
+; SM70-NEXT: setp.nan.f32 %p4, %r40, %r40;
+; SM70-NEXT: or.b32 %r44, %r40, 4194304;
+; SM70-NEXT: selp.b32 %r45, %r44, %r43, %p4;
+; SM70-NEXT: prmt.b32 %r46, %r45, %r35, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r46;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r89;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r90;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p5, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r89, %rd11;
+; SM70-NEXT: @%p5 bra $L__BB327_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd12;
+; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r3;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM70-NEXT: $L__BB327_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r47, %rs10;
+; SM70-NEXT: shl.b32 %r48, %r47, 16;
+; SM70-NEXT: mov.b32 {%rs11, %rs12}, %r91;
+; SM70-NEXT: cvt.u32.u16 %r49, %rs12;
+; SM70-NEXT: shl.b32 %r50, %r49, 16;
+; SM70-NEXT: max.f32 %r51, %r50, %r48;
+; SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
+; SM70-NEXT: add.s32 %r53, %r52, %r51;
+; SM70-NEXT: add.s32 %r54, %r53, 32767;
+; SM70-NEXT: setp.nan.f32 %p6, %r51, %r51;
+; SM70-NEXT: or.b32 %r55, %r51, 4194304;
+; SM70-NEXT: selp.b32 %r56, %r55, %r54, %p6;
+; SM70-NEXT: cvt.u32.u16 %r57, %rs9;
+; SM70-NEXT: shl.b32 %r58, %r57, 16;
+; SM70-NEXT: cvt.u32.u16 %r59, %rs11;
+; SM70-NEXT: shl.b32 %r60, %r59, 16;
+; SM70-NEXT: max.f32 %r61, %r60, %r58;
+; SM70-NEXT: bfe.u32 %r62, %r61, 16, 1;
+; SM70-NEXT: add.s32 %r63, %r62, %r61;
+; SM70-NEXT: add.s32 %r64, %r63, 32767;
+; SM70-NEXT: setp.nan.f32 %p7, %r61, %r61;
+; SM70-NEXT: or.b32 %r65, %r61, 4194304;
+; SM70-NEXT: selp.b32 %r66, %r65, %r64, %p7;
+; SM70-NEXT: prmt.b32 %r67, %r66, %r56, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r67;
+; SM70-NEXT: cvt.u32.u16 %r68, %rs14;
+; SM70-NEXT: shl.b32 %r69, %r68, 16;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r92;
+; SM70-NEXT: cvt.u32.u16 %r70, %rs16;
+; SM70-NEXT: shl.b32 %r71, %r70, 16;
+; SM70-NEXT: max.f32 %r72, %r71, %r69;
+; SM70-NEXT: bfe.u32 %r73, %r72, 16, 1;
+; SM70-NEXT: add.s32 %r74, %r73, %r72;
+; SM70-NEXT: add.s32 %r75, %r74, 32767;
+; SM70-NEXT: setp.nan.f32 %p8, %r72, %r72;
+; SM70-NEXT: or.b32 %r76, %r72, 4194304;
+; SM70-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; SM70-NEXT: cvt.u32.u16 %r78, %rs13;
+; SM70-NEXT: shl.b32 %r79, %r78, 16;
+; SM70-NEXT: cvt.u32.u16 %r80, %rs15;
+; SM70-NEXT: shl.b32 %r81, %r80, 16;
+; SM70-NEXT: max.f32 %r82, %r81, %r79;
+; SM70-NEXT: bfe.u32 %r83, %r82, 16, 1;
+; SM70-NEXT: add.s32 %r84, %r83, %r82;
+; SM70-NEXT: add.s32 %r85, %r84, 32767;
+; SM70-NEXT: setp.nan.f32 %p9, %r82, %r82;
+; SM70-NEXT: or.b32 %r86, %r82, 4194304;
+; SM70-NEXT: selp.b32 %r87, %r86, %r85, %p9;
+; SM70-NEXT: prmt.b32 %r88, %r87, %r77, 0x7632U;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r88;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r91;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r92;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p10, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r92}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r91, %rd21;
+; SM70-NEXT: @%p10 bra $L__BB327_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r89, %r90, %r91, %r92};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<8>;
+; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NEXT: mov.b32 %r19, %r3;
+; SM70-NEXT: @%p6 bra $L__BB328_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: $L__BB329_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM70-NEXT: mov.b32 %r16, %r1;
+; SM70-NEXT: @%p11 bra $L__BB329_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<29>;
+; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM70-NEXT: $L__BB330_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.lt.f32 %p11, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: setp.lt.f32 %p16, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB330_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<57>;
+; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM70-NEXT: $L__BB331_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.lt.f32 %p1, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.lt.f32 %p11, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.lt.f32 %p16, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB331_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: $L__BB331_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM70-NEXT: shl.b32 %r32, %r31, 16;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM70-NEXT: shl.b32 %r34, %r33, 16;
+; SM70-NEXT: setp.lt.f32 %p22, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs32, -32768;
+; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs30, -32768;
+; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM70-NEXT: shl.b32 %r36, %r35, 16;
+; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM70-NEXT: shl.b32 %r38, %r37, 16;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM70-NEXT: shl.b32 %r40, %r39, 16;
+; SM70-NEXT: setp.lt.f32 %p27, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs31, -32768;
+; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs29, -32768;
+; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM70-NEXT: shl.b32 %r42, %r41, 16;
+; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM70-NEXT: shl.b32 %r45, %r44, 16;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM70-NEXT: shl.b32 %r47, %r46, 16;
+; SM70-NEXT: setp.lt.f32 %p32, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs46, -32768;
+; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs44, -32768;
+; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM70-NEXT: shl.b32 %r49, %r48, 16;
+; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM70-NEXT: shl.b32 %r51, %r50, 16;
+; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM70-NEXT: shl.b32 %r53, %r52, 16;
+; SM70-NEXT: setp.lt.f32 %p37, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs45, -32768;
+; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs43, -32768;
+; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM70-NEXT: shl.b32 %r55, %r54, 16;
+; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB331_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<7>;
+; SM70-NEXT: .reg .b16 %rs<8>;
+; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NEXT: mov.b32 %r6, 65535;
+; SM70-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NEXT: not.b32 %r2, %r7;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NEXT: mov.b32 %r19, %r3;
+; SM70-NEXT: @%p6 bra $L__BB332_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<12>;
+; SM70-NEXT: .reg .b16 %rs<15>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: $L__BB333_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
+; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
+; SM70-NEXT: mov.b32 %r16, %r1;
+; SM70-NEXT: @%p11 bra $L__BB333_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<22>;
+; SM70-NEXT: .reg .b16 %rs<29>;
+; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT: shl.b32 %r4, %r3, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM70-NEXT: $L__BB334_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.gt.f32 %p11, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: setp.gt.f32 %p16, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB334_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM70-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<43>;
+; SM70-NEXT: .reg .b16 %rs<57>;
+; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .b64 %rd<22>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
+; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
+; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
+; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM70-NEXT: $L__BB335_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: setp.gt.f32 %p1, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
+; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
+; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: shl.b32 %r12, %r11, 16;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
+; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
+; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
+; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
+; SM70-NEXT: shl.b32 %r16, %r15, 16;
+; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
+; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
+; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: shl.b32 %r21, %r20, 16;
+; SM70-NEXT: setp.gt.f32 %p11, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
+; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
+; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
+; SM70-NEXT: shl.b32 %r23, %r22, 16;
+; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
+; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
+; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
+; SM70-NEXT: shl.b32 %r25, %r24, 16;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
+; SM70-NEXT: setp.gt.f32 %p16, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
+; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
+; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
+; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
+; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
+; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
+; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
+; SM70-NEXT: shl.b32 %r29, %r28, 16;
+; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
+; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
+; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
+; SM70-NEXT: @%p21 bra $L__BB335_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: $L__BB335_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
+; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
+; SM70-NEXT: shl.b32 %r32, %r31, 16;
+; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
+; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
+; SM70-NEXT: shl.b32 %r34, %r33, 16;
+; SM70-NEXT: setp.gt.f32 %p22, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
+; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
+; SM70-NEXT: setp.eq.b16 %p24, %rs32, 0;
+; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
+; SM70-NEXT: setp.eq.b16 %p25, %rs30, 0;
+; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
+; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
+; SM70-NEXT: shl.b32 %r36, %r35, 16;
+; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
+; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
+; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
+; SM70-NEXT: shl.b32 %r38, %r37, 16;
+; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
+; SM70-NEXT: shl.b32 %r40, %r39, 16;
+; SM70-NEXT: setp.gt.f32 %p27, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
+; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
+; SM70-NEXT: setp.eq.b16 %p29, %rs31, 0;
+; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs29, 0;
+; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
+; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
+; SM70-NEXT: shl.b32 %r42, %r41, 16;
+; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
+; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
+; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
+; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
+; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
+; SM70-NEXT: shl.b32 %r45, %r44, 16;
+; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
+; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
+; SM70-NEXT: shl.b32 %r47, %r46, 16;
+; SM70-NEXT: setp.gt.f32 %p32, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
+; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
+; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
+; SM70-NEXT: setp.eq.b16 %p34, %rs46, 0;
+; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
+; SM70-NEXT: setp.eq.b16 %p35, %rs44, 0;
+; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
+; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
+; SM70-NEXT: shl.b32 %r49, %r48, 16;
+; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
+; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
+; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
+; SM70-NEXT: shl.b32 %r51, %r50, 16;
+; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
+; SM70-NEXT: shl.b32 %r53, %r52, 16;
+; SM70-NEXT: setp.gt.f32 %p37, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
+; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
+; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
+; SM70-NEXT: setp.eq.b16 %p39, %rs45, 0;
+; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
+; SM70-NEXT: setp.eq.b16 %p40, %rs43, 0;
+; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
+; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
+; SM70-NEXT: shl.b32 %r55, %r54, 16;
+; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
+; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
+; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
+; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
+; SM70-NEXT: @%p42 bra $L__BB335_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_monotonic_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB336_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB336_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_monotonic_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB337_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB337_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_monotonic_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB338_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB338_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_monotonic_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB339_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB339_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_acquire_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB340_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB340_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_acquire_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB341_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB341_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_acquire_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB342_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB342_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_acquire_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB343_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB343_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_release_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB344_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB344_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_release_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB345_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_release_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB346_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB346_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_release_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB347_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB347_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: $L__BB348_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: add.s32 %r10, %r15, %r4;
+; SM70-NEXT: and.b32 %r11, %r10, %r2;
+; SM70-NEXT: and.b32 %r12, %r15, %r3;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM70-NEXT: mov.b32 %r15, %r5;
+; SM70-NEXT: @%p1 bra $L__BB348_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<12>;
+; SM70-NEXT: .reg .b32 %r<16>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM70-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r15, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
+; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
+; SM70-NEXT: and.b16 %rs9, %rs8, 255;
+; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM70-NEXT: shl.b32 %r11, %r10, %r1;
+; SM70-NEXT: and.b32 %r12, %r15, %r2;
+; SM70-NEXT: or.b32 %r13, %r12, %r11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM70-NEXT: mov.b32 %r15, %r3;
+; SM70-NEXT: @%p1 bra $L__BB349_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r14, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r14;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: $L__BB350_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM70-NEXT: mov.b32 %r18, %r1;
+; SM70-NEXT: @%p1 bra $L__BB350_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: add_seq_cst_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<35>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM70-NEXT: $L__BB351_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB351_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_monotonic_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_monotonic_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_monotonic_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_monotonic_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_acquire_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_acquire_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_acquire_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_acquire_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_release_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_release_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_release_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<3>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<5>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: add_seq_cst_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB368_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB369_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB370_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_monotonic_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB371_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_acquire_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM70-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB372_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_acquire_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB373_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_acquire_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB374_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_acquire_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB375_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_release_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB376_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_release_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB377_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB377_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_release_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB378_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB378_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_release_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB379_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB379_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v1i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: and.b32 %r8, %r7, 3;
+; SM70-NEXT: shl.b32 %r1, %r8, 3;
+; SM70-NEXT: mov.b32 %r9, 255;
+; SM70-NEXT: shl.b32 %r2, %r9, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: shl.b32 %r4, %r6, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: $L__BB380_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
+; SM70-NEXT: @%p1 bra $L__BB380_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b8 [func_retval0], %r15;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: and.b32 %r6, %r5, 3;
+; SM70-NEXT: shl.b32 %r1, %r6, 3;
+; SM70-NEXT: mov.b32 %r7, 65535;
+; SM70-NEXT: shl.b32 %r8, %r7, %r1;
+; SM70-NEXT: not.b32 %r2, %r8;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: $L__BB381_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: shr.u32 %r9, %r18, %r1;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM70-NEXT: and.b32 %r11, %r10, %r4;
+; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM70-NEXT: shl.b32 %r14, %r13, %r1;
+; SM70-NEXT: and.b32 %r15, %r18, %r2;
+; SM70-NEXT: or.b32 %r16, %r15, %r14;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: @%p1 bra $L__BB381_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB382_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r2;
+; SM70-NEXT: xor.b32 %r4, %r3, -1;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM70-NEXT: mov.b32 %r5, %r1;
+; SM70-NEXT: @%p1 bra $L__BB382_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b64 %rd<12>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM70-NEXT: $L__BB383_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r8, %r2;
+; SM70-NEXT: and.b32 %r4, %r7, %r1;
+; SM70-NEXT: xor.b32 %r5, %r4, -1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM70-NEXT: xor.b32 %r6, %r3, -1;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM70-NEXT: @%p1 bra $L__BB383_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB384_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB385_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB386_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB386_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB386_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_monotonic_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB387_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB387_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB387_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB387_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB387_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB387_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB387_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_acquire_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM70-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB388_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_acquire_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB389_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_acquire_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB390_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB390_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_acquire_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB391_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB391_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB391_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB391_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB391_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB391_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB391_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_release_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB392_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_release_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB393_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_release_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB394_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB394_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_release_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB395_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB395_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB395_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB395_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB395_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB395_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB395_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<6>;
+; SM70-NEXT: .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM70-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: and.b32 %r3, %r5, %r1;
+; SM70-NEXT: not.b32 %r4, %r3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM70-NEXT: mov.b32 %r5, %r2;
+; SM70-NEXT: @%p1 bra $L__BB396_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<2>;
+; SM70-NEXT: .reg .b32 %r<7>;
+; SM70-NEXT: .reg .b64 %rd<13>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM70-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r3, %r6, %r2;
+; SM70-NEXT: and.b32 %r4, %r5, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB397_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<3>;
+; SM70-NEXT: .reg .b32 %r<13>;
+; SM70-NEXT: .reg .b64 %rd<24>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
+; SM70-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r5, %r10, %r2;
+; SM70-NEXT: and.b32 %r6, %r9, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB398_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
+; SM70-NEXT: $L__BB398_3: // %atomicrmw.start2
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r7, %r12, %r4;
+; SM70-NEXT: and.b32 %r8, %r11, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB398_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end1
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM70-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM70: {
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<25>;
+; SM70-NEXT: .reg .b64 %rd<46>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
+; SM70-NEXT: $L__BB399_1: // %atomicrmw.start
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: and.b32 %r9, %r18, %r6;
+; SM70-NEXT: and.b32 %r10, %r17, %r5;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
+; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM70-NEXT: not.b64 %rd11, %rd10;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
+; SM70-NEXT: @%p1 bra $L__BB399_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
+; SM70-NEXT: $L__BB399_3: // %atomicrmw.start6
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
+; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
+; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
+; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
+; SM70-NEXT: and.b32 %r11, %r20, %r8;
+; SM70-NEXT: and.b32 %r12, %r19, %r7;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
+; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM70-NEXT: not.b64 %rd22, %rd21;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
+; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
+; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
+; SM70-NEXT: @%p2 bra $L__BB399_3;
+; SM70-NEXT: // %bb.4: // %atomicrmw.end5
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
+; SM70-NEXT: $L__BB399_5: // %atomicrmw.start16
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
+; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
+; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
+; SM70-NEXT: and.b32 %r13, %r22, %r2;
+; SM70-NEXT: and.b32 %r14, %r21, %r1;
+; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
+; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM70-NEXT: not.b64 %rd33, %rd32;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
+; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
+; SM70-NEXT: @%p3 bra $L__BB399_5;
+; SM70-NEXT: // %bb.6: // %atomicrmw.end15
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
+; SM70-NEXT: $L__BB399_7: // %atomicrmw.start22
+; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
+; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
+; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
+; SM70-NEXT: and.b32 %r15, %r24, %r4;
+; SM70-NEXT: and.b32 %r16, %r23, %r3;
+; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
+; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
+; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
+; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
+; SM70-NEXT: not.b64 %rd44, %rd43;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
+; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
+; SM70-NEXT: @%p4 bra $L__BB399_7;
+; SM70-NEXT: // %bb.8: // %atomicrmw.end21
+; SM70-NEXT: fence.acq_rel.cta;
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM70-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
new file mode 100644
index 0000000000000..2cf9406bd0447
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -0,0 +1,18160 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 255;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB0_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB1_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<14>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r9, %r8, %r1;
+; SM90-NEXT: not.b32 %r2, %r9;
+; SM90-NEXT: shl.b32 %r3, %r5, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1];
+; SM90-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r13, %r2;
+; SM90-NEXT: or.b32 %r11, %r10, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11;
+; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13;
+; SM90-NEXT: mov.b32 %r13, %r4;
+; SM90-NEXT: @%p1 bra $L__BB4_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r12, %r4, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 amt, dst;
+; SM90-NEXT: mov.b128 amt, {%rd2, %rd3};
+; SM90-NEXT: atom.acq_rel.cta.global.exch.b128 dst, [%rd1], amt;
+; SM90-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: xchg_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB16_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB16_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB17_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB17_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB18_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB18_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB19_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB19_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB20_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB20_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB21_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB21_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB22_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: add.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB22_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: add_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB23_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: add.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: add.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: add.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB23_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: add_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB32_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB32_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<12>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: $L__BB33_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: sub.s16 %rs7, %rs4, %rs6;
+; SM90-NEXT: sub.s16 %rs8, %rs3, %rs5;
+; SM90-NEXT: and.b16 %rs9, %rs8, 255;
+; SM90-NEXT: shl.b16 %rs10, %rs7, 8;
+; SM90-NEXT: or.b16 %rs11, %rs9, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs11;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB33_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB34_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB34_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: sub_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB35_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: sub.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: sub.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: sub.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: sub.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: sub.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB35_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB36_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB36_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB37_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB37_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM90-NEXT: $L__BB38_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r5;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r6;
+; SM90-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM90-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM90-NEXT: mov.b32 %r4, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB38_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: sub_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r2;
+; SM90-NEXT: $L__BB39_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs6, %rs3, %rs1;
+; SM90-NEXT: mov.b32 %r5, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM90-NEXT: sub.s16 %rs11, %rs10, %rs8;
+; SM90-NEXT: sub.s16 %rs12, %rs9, %rs7;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r3;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r11;
+; SM90-NEXT: sub.s16 %rs17, %rs16, %rs14;
+; SM90-NEXT: sub.s16 %rs18, %rs15, %rs13;
+; SM90-NEXT: mov.b32 %r7, {%rs18, %rs17};
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r4;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r12;
+; SM90-NEXT: sub.s16 %rs23, %rs22, %rs20;
+; SM90-NEXT: sub.s16 %rs24, %rs21, %rs19;
+; SM90-NEXT: mov.b32 %r8, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB39_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<4>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r2, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r3, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
+; SM90-NEXT: neg.s32 %r5, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r5, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
+; SM90-NEXT: neg.s32 %r7, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
+; SM90-NEXT: neg.s32 %r9, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
+; SM90-NEXT: neg.s32 %r11, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: sub_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: neg.s32 %r9, %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
+; SM90-NEXT: neg.s32 %r11, %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
+; SM90-NEXT: neg.s32 %r13, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
+; SM90-NEXT: neg.s32 %r15, %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
+; SM90-NEXT: neg.s32 %r17, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
+; SM90-NEXT: neg.s32 %r19, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
+; SM90-NEXT: neg.s32 %r21, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
+; SM90-NEXT: neg.s32 %r23, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<5>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd3, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<8>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd4, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
+; SM90-NEXT: neg.s64 %rd6, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd6, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
+; SM90-NEXT: neg.s64 %rd8, %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
+; SM90-NEXT: neg.s64 %rd10, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
+; SM90-NEXT: neg.s64 %rd12, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: sub_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: neg.s64 %rd10, %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
+; SM90-NEXT: neg.s64 %rd12, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
+; SM90-NEXT: neg.s64 %rd14, %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
+; SM90-NEXT: neg.s64 %rd16, %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
+; SM90-NEXT: neg.s64 %rd18, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
+; SM90-NEXT: neg.s64 %rd20, %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
+; SM90-NEXT: neg.s64 %rd22, %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
+; SM90-NEXT: neg.s64 %rd24, %rd3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: mov.b32 %r5, 255;
+; SM90-NEXT: shl.b32 %r6, %r5, %r4;
+; SM90-NEXT: not.b32 %r7, %r6;
+; SM90-NEXT: shl.b32 %r8, %r1, %r4;
+; SM90-NEXT: or.b32 %r9, %r8, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: mov.b32 %r4, 65535;
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: not.b32 %r6, %r5;
+; SM90-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r8, %r7, %r3;
+; SM90-NEXT: or.b32 %r9, %r8, %r6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB50_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB50_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: and_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB51_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: and.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB51_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<12>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: mov.b32 %r5, 65535;
+; SM90-NEXT: shl.b32 %r6, %r5, %r4;
+; SM90-NEXT: not.b32 %r7, %r6;
+; SM90-NEXT: shl.b32 %r8, %r1, %r4;
+; SM90-NEXT: or.b32 %r9, %r8, %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
+; SM90-NEXT: shr.u32 %r11, %r10, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB53_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB53_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB54_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: and.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB54_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: and_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB55_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: and.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: and.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: and.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB55_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: and_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: and_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB66_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB66_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB67_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB67_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 65535;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB68_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB68_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB69_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB69_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB70_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB70_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: nand_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB71_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r16, %r4;
+; SM90-NEXT: and.b32 %r6, %r15, %r3;
+; SM90-NEXT: and.b32 %r7, %r14, %r2;
+; SM90-NEXT: and.b32 %r8, %r13, %r1;
+; SM90-NEXT: xor.b32 %r9, %r8, -1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: xor.b32 %r10, %r7, -1;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: xor.b32 %r11, %r6, -1;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: xor.b32 %r12, %r5, -1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB71_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB72_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB75_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB75_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd4, %rd6, %rd1;
+; SM90-NEXT: not.b64 %rd5, %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd7, %rd15, %rd5;
+; SM90-NEXT: and.b64 %rd8, %rd14, %rd4;
+; SM90-NEXT: not.b64 %rd9, %rd8;
+; SM90-NEXT: not.b64 %rd10, %rd7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd11, %rd27, %rd2;
+; SM90-NEXT: and.b64 %rd12, %rd26, %rd1;
+; SM90-NEXT: not.b64 %rd13, %rd12;
+; SM90-NEXT: not.b64 %rd14, %rd11;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB78_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB78_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd19, %rd29, %rd4;
+; SM90-NEXT: and.b64 %rd20, %rd28, %rd3;
+; SM90-NEXT: not.b64 %rd21, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB78_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: nand_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: and.b64 %rd19, %rd10, %rd6;
+; SM90-NEXT: and.b64 %rd20, %rd9, %rd5;
+; SM90-NEXT: not.b64 %rd21, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB79_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB79_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: and.b64 %rd27, %rd12, %rd8;
+; SM90-NEXT: and.b64 %rd28, %rd11, %rd7;
+; SM90-NEXT: not.b64 %rd29, %rd28;
+; SM90-NEXT: not.b64 %rd30, %rd27;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
+; SM90-NEXT: @%p2 bra $L__BB79_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB79_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd35, %rd55, %rd2;
+; SM90-NEXT: and.b64 %rd36, %rd54, %rd1;
+; SM90-NEXT: not.b64 %rd37, %rd36;
+; SM90-NEXT: not.b64 %rd38, %rd35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB79_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB79_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b64 %rd43, %rd57, %rd4;
+; SM90-NEXT: and.b64 %rd44, %rd56, %rd3;
+; SM90-NEXT: not.b64 %rd45, %rd44;
+; SM90-NEXT: not.b64 %rd46, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB79_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB82_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: or_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: or.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB83_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB85_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB85_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB86_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: or.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB86_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: or_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB87_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: or.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: or.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: or.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: or.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB87_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: or_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: or_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM90-NEXT: and.b32 %r2, %r1, 3;
+; SM90-NEXT: shl.b32 %r3, %r2, 3;
+; SM90-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r5, %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB98_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB98_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: xor_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB99_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: xor.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB99_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<8>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd2, %rd1, -4;
+; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
+; SM90-NEXT: and.b32 %r3, %r2, 3;
+; SM90-NEXT: shl.b32 %r4, %r3, 3;
+; SM90-NEXT: shl.b32 %r5, %r1, %r4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
+; SM90-NEXT: shr.u32 %r7, %r6, %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB101_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB101_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB102_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: xor.b32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB102_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: xor_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB103_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: xor.b32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: xor.b32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: xor.b32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: xor.b32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB103_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: xor_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: xor_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB112_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM90-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB112_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<14>;
+; SM90-NEXT: .reg .b32 %r<20>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
+; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
+; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
+; SM90-NEXT: $L__BB113_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r19, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM90-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM90-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM90-NEXT: mov.b32 %r11, {%rs3, %rs4};
+; SM90-NEXT: max.s16x2 %r13, %r11, %r12;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r13;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: and.b16 %rs12, %rs9, 255;
+; SM90-NEXT: or.b16 %rs13, %rs12, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs13;
+; SM90-NEXT: shl.b32 %r15, %r14, %r1;
+; SM90-NEXT: and.b32 %r16, %r19, %r2;
+; SM90-NEXT: or.b32 %r17, %r16, %r15;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
+; SM90-NEXT: mov.b32 %r19, %r3;
+; SM90-NEXT: @%p1 bra $L__BB113_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r18, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM90-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM90-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM90-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM90-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM90-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM90-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM90-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p5 bra $L__BB114_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: max_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<51>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: $L__BB115_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM90-NEXT: setp.gt.s32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM90-NEXT: setp.gt.s32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM90-NEXT: setp.gt.s32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM90-NEXT: setp.gt.s32 %p8, %r18, %r17;
+; SM90-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM90-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM90-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM90-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM90-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM90-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM90-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM90-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM90-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM90-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM90-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM90-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM90-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM90-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM90-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM90-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM90-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB115_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB116_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB116_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB117_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB117_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB118_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: max.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: max.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB118_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: max_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB119_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: max.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: max.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: max.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: max.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB119_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: max_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: max_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB128_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs2, %r8;
+; SM90-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p1 bra $L__BB128_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<14>;
+; SM90-NEXT: .reg .b32 %r<20>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: cvt.s16.s8 %rs7, %rs6;
+; SM90-NEXT: cvt.s16.s8 %rs8, %rs5;
+; SM90-NEXT: mov.b32 %r12, {%rs8, %rs7};
+; SM90-NEXT: $L__BB129_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r19, %r1;
+; SM90-NEXT: cvt.s8.s32 %rs3, %r9;
+; SM90-NEXT: bfe.s32 %r10, %r9, 8, 8;
+; SM90-NEXT: cvt.s8.s32 %rs4, %r10;
+; SM90-NEXT: mov.b32 %r11, {%rs3, %rs4};
+; SM90-NEXT: min.s16x2 %r13, %r11, %r12;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r13;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: and.b16 %rs12, %rs9, 255;
+; SM90-NEXT: or.b16 %rs13, %rs12, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs13;
+; SM90-NEXT: shl.b32 %r15, %r14, %r1;
+; SM90-NEXT: and.b32 %r16, %r19, %r2;
+; SM90-NEXT: or.b32 %r17, %r16, %r15;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r19;
+; SM90-NEXT: mov.b32 %r19, %r3;
+; SM90-NEXT: @%p1 bra $L__BB129_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r18, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r15, %r26, 0, 0x7773U;
+; SM90-NEXT: selp.b32 %r16, %r15, %r14, %p4;
+; SM90-NEXT: prmt.b32 %r17, %r26, 0, 0x7772U;
+; SM90-NEXT: selp.b32 %r18, %r17, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r19, %r18, %r16, 0x3340U;
+; SM90-NEXT: prmt.b32 %r20, %r26, 0, 0x7771U;
+; SM90-NEXT: selp.b32 %r21, %r20, %r12, %p2;
+; SM90-NEXT: prmt.b32 %r22, %r26, 0, 0x7770U;
+; SM90-NEXT: selp.b32 %r23, %r22, %r11, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r21, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r19, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p5 bra $L__BB130_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: min_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<51>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
+; SM90-NEXT: $L__BB131_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r8, %r50, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r10, %r50, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x8880U;
+; SM90-NEXT: prmt.b32 %r12, %r49, 0, 0x8880U;
+; SM90-NEXT: setp.le.s32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x9991U;
+; SM90-NEXT: prmt.b32 %r14, %r49, 0, 0x9991U;
+; SM90-NEXT: setp.le.s32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0xaaa2U;
+; SM90-NEXT: prmt.b32 %r16, %r49, 0, 0xaaa2U;
+; SM90-NEXT: setp.le.s32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0xbbb3U;
+; SM90-NEXT: prmt.b32 %r18, %r49, 0, 0xbbb3U;
+; SM90-NEXT: setp.le.s32 %p8, %r18, %r17;
+; SM90-NEXT: prmt.b32 %r19, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r19;
+; SM90-NEXT: prmt.b32 %r20, %r50, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r20;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r50, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r22;
+; SM90-NEXT: prmt.b32 %r23, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r23;
+; SM90-NEXT: prmt.b32 %r24, %r50, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r24;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r50, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r26;
+; SM90-NEXT: prmt.b32 %r27, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r27;
+; SM90-NEXT: prmt.b32 %r28, %r49, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r29;
+; SM90-NEXT: prmt.b32 %r30, %r49, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r30;
+; SM90-NEXT: prmt.b32 %r31, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r31;
+; SM90-NEXT: prmt.b32 %r32, %r49, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r32;
+; SM90-NEXT: prmt.b32 %r33, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r33;
+; SM90-NEXT: prmt.b32 %r34, %r49, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r34;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r35, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r36, %rs18;
+; SM90-NEXT: prmt.b32 %r37, %r36, %r35, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r38, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r39, %rs20;
+; SM90-NEXT: prmt.b32 %r40, %r39, %r38, 0x3340U;
+; SM90-NEXT: prmt.b32 %r41, %r40, %r37, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r42, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r43, %rs22;
+; SM90-NEXT: prmt.b32 %r44, %r43, %r42, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r45, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r46, %rs24;
+; SM90-NEXT: prmt.b32 %r47, %r46, %r45, 0x3340U;
+; SM90-NEXT: prmt.b32 %r48, %r47, %r44, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r48;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r50}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r49, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB131_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB132_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB132_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB133_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.s16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB133_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB134_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: min.s16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: min.s16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB134_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: min_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB135_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: min.s16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: min.s16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: min.s16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: min.s16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB135_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: min_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: min_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB144_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB144_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB145_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: max.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB145_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB146_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM90-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM90-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM90-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM90-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p5 bra $L__BB146_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: umax_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: $L__BB147_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB147_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB148_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB148_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB149_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB149_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB150_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: max.u16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: max.u16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB150_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: umax_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB151_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: max.u16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: max.u16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: max.u16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: max.u16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB151_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: umax_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: umax_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB160_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: min.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB160_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB161_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: min.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB161_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB162_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
+; SM90-NEXT: selp.b32 %r12, %r8, %r7, %p3;
+; SM90-NEXT: prmt.b32 %r13, %r12, %r11, 0x3340U;
+; SM90-NEXT: selp.b32 %r14, %r6, %r5, %p2;
+; SM90-NEXT: selp.b32 %r15, %r4, %r3, %p1;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r14, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p5 bra $L__BB162_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: umin_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: $L__BB163_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.le.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.le.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.le.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r18;
+; SM90-NEXT: selp.b16 %rs17, %rs16, %rs15, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, %rs14, %rs13, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, %rs12, %rs11, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, %rs10, %rs9, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, %rs8, %rs7, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, %rs6, %rs5, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, %rs4, %rs3, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, %rs2, %rs1, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB163_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB164_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB164_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB165_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.u16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB165_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB166_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: min.u16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: min.u16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB166_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: umin_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB167_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: min.u16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: min.u16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: min.u16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: min.u16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB167_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: umin_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: umin_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB176_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: add.s16 %rs4, %rs2, 1;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: selp.b16 %rs5, 0, %rs4, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p2 bra $L__BB176_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<15>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM90-NEXT: $L__BB177_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b16 %rs5, %rs3, 255;
+; SM90-NEXT: mov.b32 %r11, 65537;
+; SM90-NEXT: add.s16x2 %r12, %r10, %r11;
+; SM90-NEXT: setp.ge.u16 %p1, %rs4, %rs7;
+; SM90-NEXT: setp.ge.u16 %p2, %rs5, %rs6;
+; SM90-NEXT: mov.b32 {%rs8, %rs9}, %r12;
+; SM90-NEXT: selp.b16 %rs10, 0, %rs8, %p2;
+; SM90-NEXT: selp.b16 %rs11, 0, %rs9, %p1;
+; SM90-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM90-NEXT: and.b16 %rs13, %rs10, 255;
+; SM90-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs14;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p3 bra $L__BB177_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<23>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB178_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, 1;
+; SM90-NEXT: prmt.b32 %r4, %r22, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, 1;
+; SM90-NEXT: prmt.b32 %r5, %r22, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, 1;
+; SM90-NEXT: prmt.b32 %r6, %r22, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, 1;
+; SM90-NEXT: setp.ge.u32 %p1, %r3, %r7;
+; SM90-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM90-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM90-NEXT: setp.ge.u32 %p4, %r6, %r10;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM90-NEXT: selp.b32 %r12, 0, %r11, %p4;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs6;
+; SM90-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; SM90-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r16, %rs4;
+; SM90-NEXT: selp.b32 %r17, 0, %r16, %p2;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs2;
+; SM90-NEXT: selp.b32 %r19, 0, %r18, %p1;
+; SM90-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM90-NEXT: mov.b32 %r22, %r1;
+; SM90-NEXT: @%p5 bra $L__BB178_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: $L__BB179_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, 1;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, 1;
+; SM90-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, 1;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, 1;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM90-NEXT: add.s16 %rs10, %rs9, 1;
+; SM90-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM90-NEXT: add.s16 %rs12, %rs11, 1;
+; SM90-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM90-NEXT: add.s16 %rs14, %rs13, 1;
+; SM90-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM90-NEXT: add.s16 %rs16, %rs15, 1;
+; SM90-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM90-NEXT: setp.ge.u32 %p2, %r4, %r12;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r5, %r13;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r6, %r14;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p5, %r7, %r15;
+; SM90-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p6, %r8, %r16;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p7, %r9, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p8, %r10, %r18;
+; SM90-NEXT: selp.b16 %rs17, 0, %rs16, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs17;
+; SM90-NEXT: selp.b16 %rs18, 0, %rs14, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs19, 0, %rs12, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs19;
+; SM90-NEXT: selp.b16 %rs20, 0, %rs10, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs20;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs21, 0, %rs8, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs21;
+; SM90-NEXT: selp.b16 %rs22, 0, %rs6, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs22;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs23, 0, %rs4, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs23;
+; SM90-NEXT: selp.b16 %rs24, 0, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB179_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB180_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: add.s16 %rs3, %rs2, 1;
+; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, 0, %rs3, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p2 bra $L__BB180_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r6, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB181_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 %r3, 65537;
+; SM90-NEXT: add.s16x2 %r4, %r6, %r3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r4;
+; SM90-NEXT: selp.b16 %rs7, 0, %rs6, %p2;
+; SM90-NEXT: selp.b16 %rs8, 0, %rs5, %p1;
+; SM90-NEXT: mov.b32 %r5, {%rs8, %rs7};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r6, %r5;
+; SM90-NEXT: setp.ne.b32 %p3, %r1, %r6;
+; SM90-NEXT: mov.b32 %r6, %r1;
+; SM90-NEXT: @%p3 bra $L__BB181_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<10>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r9}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT: $L__BB182_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r9;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 %r3, 65537;
+; SM90-NEXT: add.s16x2 %r4, %r9, %r3;
+; SM90-NEXT: add.s16x2 %r5, %r8, %r3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r9;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r8;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r5;
+; SM90-NEXT: selp.b16 %rs11, 0, %rs10, %p4;
+; SM90-NEXT: selp.b16 %rs12, 0, %rs9, %p3;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r6;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r4;
+; SM90-NEXT: selp.b16 %rs15, 0, %rs14, %p2;
+; SM90-NEXT: selp.b16 %rs16, 0, %rs13, %p1;
+; SM90-NEXT: mov.b32 %r7, {%rs16, %rs15};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r7;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r9}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r8, %rd11;
+; SM90-NEXT: @%p5 bra $L__BB182_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r8, %r9};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r17}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r16, %rd4;
+; SM90-NEXT: mov.b64 {_, %r15}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r14, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT: $L__BB183_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r15;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r17;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 %r5, 65537;
+; SM90-NEXT: add.s16x2 %r6, %r17, %r5;
+; SM90-NEXT: add.s16x2 %r7, %r16, %r5;
+; SM90-NEXT: add.s16x2 %r8, %r15, %r5;
+; SM90-NEXT: add.s16x2 %r9, %r14, %r5;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r17;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r16;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r15;
+; SM90-NEXT: setp.ge.u16 %p5, %rs11, %rs9;
+; SM90-NEXT: setp.ge.u16 %p6, %rs12, %rs10;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r1;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r14;
+; SM90-NEXT: setp.ge.u16 %p7, %rs15, %rs13;
+; SM90-NEXT: setp.ge.u16 %p8, %rs16, %rs14;
+; SM90-NEXT: mov.b32 {%rs17, %rs18}, %r9;
+; SM90-NEXT: selp.b16 %rs19, 0, %rs18, %p8;
+; SM90-NEXT: selp.b16 %rs20, 0, %rs17, %p7;
+; SM90-NEXT: mov.b32 %r10, {%rs20, %rs19};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r10;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r8;
+; SM90-NEXT: selp.b16 %rs23, 0, %rs22, %p6;
+; SM90-NEXT: selp.b16 %rs24, 0, %rs21, %p5;
+; SM90-NEXT: mov.b32 %r11, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: mov.b32 {%rs25, %rs26}, %r7;
+; SM90-NEXT: selp.b16 %rs27, 0, %rs26, %p4;
+; SM90-NEXT: selp.b16 %rs28, 0, %rs25, %p3;
+; SM90-NEXT: mov.b32 %r12, {%rs28, %rs27};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r12;
+; SM90-NEXT: mov.b32 {%rs29, %rs30}, %r6;
+; SM90-NEXT: selp.b16 %rs31, 0, %rs30, %p2;
+; SM90-NEXT: selp.b16 %rs32, 0, %rs29, %p1;
+; SM90-NEXT: mov.b32 %r13, {%rs32, %rs31};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r13;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r17}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r16, %rd22;
+; SM90-NEXT: mov.b64 {_, %r15}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r14, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
+; SM90-NEXT: @%p9 bra $L__BB183_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r14, %r15, %r16, %r17};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd4, %rd6, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM90-NEXT: selp.b64 %rd5, 0, %rd4, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p2 bra $L__BB188_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB189_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd7, %rd15, 1;
+; SM90-NEXT: add.s64 %rd8, %rd14, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd15, %rd5;
+; SM90-NEXT: setp.ge.u64 %p2, %rd14, %rd4;
+; SM90-NEXT: selp.b64 %rd9, 0, %rd8, %p2;
+; SM90-NEXT: selp.b64 %rd10, 0, %rd7, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB189_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<7>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB190_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd11, %rd27, 1;
+; SM90-NEXT: add.s64 %rd12, %rd26, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd27, %rd2;
+; SM90-NEXT: setp.ge.u64 %p2, %rd26, %rd1;
+; SM90-NEXT: selp.b64 %rd13, 0, %rd12, %p2;
+; SM90-NEXT: selp.b64 %rd14, 0, %rd11, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p3 bra $L__BB190_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB190_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd19, %rd29, 1;
+; SM90-NEXT: add.s64 %rd20, %rd28, 1;
+; SM90-NEXT: setp.ge.u64 %p4, %rd29, %rd4;
+; SM90-NEXT: setp.ge.u64 %p5, %rd28, %rd3;
+; SM90-NEXT: selp.b64 %rd21, 0, %rd20, %p5;
+; SM90-NEXT: selp.b64 %rd22, 0, %rd19, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p6 bra $L__BB190_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<13>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB191_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: add.s64 %rd19, %rd10, 1;
+; SM90-NEXT: add.s64 %rd20, %rd9, 1;
+; SM90-NEXT: setp.ge.u64 %p1, %rd10, %rd6;
+; SM90-NEXT: setp.ge.u64 %p2, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, 0, %rd20, %p2;
+; SM90-NEXT: selp.b64 %rd22, 0, %rd19, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
+; SM90-NEXT: @%p3 bra $L__BB191_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB191_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: add.s64 %rd27, %rd12, 1;
+; SM90-NEXT: add.s64 %rd28, %rd11, 1;
+; SM90-NEXT: setp.ge.u64 %p4, %rd12, %rd8;
+; SM90-NEXT: setp.ge.u64 %p5, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd29, 0, %rd28, %p5;
+; SM90-NEXT: selp.b64 %rd30, 0, %rd27, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
+; SM90-NEXT: @%p6 bra $L__BB191_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB191_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd35, %rd55, 1;
+; SM90-NEXT: add.s64 %rd36, %rd54, 1;
+; SM90-NEXT: setp.ge.u64 %p7, %rd55, %rd2;
+; SM90-NEXT: setp.ge.u64 %p8, %rd54, %rd1;
+; SM90-NEXT: selp.b64 %rd37, 0, %rd36, %p8;
+; SM90-NEXT: selp.b64 %rd38, 0, %rd35, %p7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p9 bra $L__BB191_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB191_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd43, %rd57, 1;
+; SM90-NEXT: add.s64 %rd44, %rd56, 1;
+; SM90-NEXT: setp.ge.u64 %p10, %rd57, %rd4;
+; SM90-NEXT: setp.ge.u64 %p11, %rd56, %rd3;
+; SM90-NEXT: selp.b64 %rd45, 0, %rd44, %p11;
+; SM90-NEXT: selp.b64 %rd46, 0, %rd43, %p10;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p12 bra $L__BB191_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB192_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs3, 0;
+; SM90-NEXT: setp.gt.u16 %p2, %rs3, %rs1;
+; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p3 bra $L__BB192_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs8, %rs9}, %r4;
+; SM90-NEXT: $L__BB193_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: and.b16 %rs4, %rs3, 255;
+; SM90-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: add.s16 %rs7, %rs3, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs4, 0;
+; SM90-NEXT: setp.gt.u16 %p3, %rs5, %rs9;
+; SM90-NEXT: setp.gt.u16 %p4, %rs4, %rs8;
+; SM90-NEXT: selp.b16 %rs10, %rs8, %rs7, %p4;
+; SM90-NEXT: selp.b16 %rs11, %rs8, %rs10, %p2;
+; SM90-NEXT: selp.b16 %rs12, %rs9, %rs6, %p3;
+; SM90-NEXT: selp.b16 %rs13, %rs9, %rs12, %p1;
+; SM90-NEXT: shl.b16 %rs14, %rs13, 8;
+; SM90-NEXT: and.b16 %rs15, %rs11, 255;
+; SM90-NEXT: or.b16 %rs16, %rs15, %rs14;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs16;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p5, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p5 bra $L__BB193_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<27>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
+; SM90-NEXT: $L__BB194_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, -1;
+; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, -1;
+; SM90-NEXT: prmt.b32 %r5, %r26, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, -1;
+; SM90-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM90-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM90-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM90-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM90-NEXT: setp.gt.u32 %p5, %r3, %r7;
+; SM90-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM90-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM90-NEXT: setp.gt.u32 %p8, %r6, %r10;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
+; SM90-NEXT: selp.b32 %r12, %r10, %r11, %p8;
+; SM90-NEXT: selp.b32 %r13, %r10, %r12, %p4;
+; SM90-NEXT: cvt.u32.u16 %r14, %rs6;
+; SM90-NEXT: selp.b32 %r15, %r9, %r14, %p7;
+; SM90-NEXT: selp.b32 %r16, %r9, %r15, %p3;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r13, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs4;
+; SM90-NEXT: selp.b32 %r19, %r8, %r18, %p6;
+; SM90-NEXT: selp.b32 %r20, %r8, %r19, %p2;
+; SM90-NEXT: cvt.u32.u16 %r21, %rs2;
+; SM90-NEXT: selp.b32 %r22, %r7, %r21, %p5;
+; SM90-NEXT: selp.b32 %r23, %r7, %r22, %p1;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r17, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
+; SM90-NEXT: setp.ne.b32 %p9, %r1, %r26;
+; SM90-NEXT: mov.b32 %r26, %r1;
+; SM90-NEXT: @%p9 bra $L__BB194_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<18>;
+; SM90-NEXT: .reg .b16 %rs<41>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs18, %r12;
+; SM90-NEXT: $L__BB195_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: add.s16 %rs2, %rs1, -1;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r4;
+; SM90-NEXT: add.s16 %rs4, %rs3, -1;
+; SM90-NEXT: prmt.b32 %r5, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r5;
+; SM90-NEXT: add.s16 %rs6, %rs5, -1;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: add.s16 %rs8, %rs7, -1;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r7;
+; SM90-NEXT: add.s16 %rs10, %rs9, -1;
+; SM90-NEXT: prmt.b32 %r8, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r8;
+; SM90-NEXT: add.s16 %rs12, %rs11, -1;
+; SM90-NEXT: prmt.b32 %r9, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r9;
+; SM90-NEXT: add.s16 %rs14, %rs13, -1;
+; SM90-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r10;
+; SM90-NEXT: add.s16 %rs16, %rs15, -1;
+; SM90-NEXT: setp.eq.b32 %p1, %r3, 0;
+; SM90-NEXT: setp.eq.b32 %p2, %r4, 0;
+; SM90-NEXT: setp.eq.b32 %p3, %r5, 0;
+; SM90-NEXT: setp.eq.b32 %p4, %r6, 0;
+; SM90-NEXT: setp.eq.b32 %p5, %r7, 0;
+; SM90-NEXT: setp.eq.b32 %p6, %r8, 0;
+; SM90-NEXT: setp.eq.b32 %p7, %r9, 0;
+; SM90-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM90-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM90-NEXT: setp.gt.u32 %p10, %r4, %r12;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p11, %r5, %r13;
+; SM90-NEXT: prmt.b32 %r14, %r2, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p12, %r6, %r14;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7770U;
+; SM90-NEXT: setp.gt.u32 %p13, %r7, %r15;
+; SM90-NEXT: prmt.b32 %r16, %r1, 0, 0x7771U;
+; SM90-NEXT: setp.gt.u32 %p14, %r8, %r16;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7772U;
+; SM90-NEXT: setp.gt.u32 %p15, %r9, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
+; SM90-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs21, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs24, %r18;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM90-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs26;
+; SM90-NEXT: selp.b16 %rs27, %rs23, %rs14, %p15;
+; SM90-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs28;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM90-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs30;
+; SM90-NEXT: selp.b16 %rs31, %rs21, %rs10, %p13;
+; SM90-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs32;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM90-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs34;
+; SM90-NEXT: selp.b16 %rs35, %rs19, %rs6, %p11;
+; SM90-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs36;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM90-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs38;
+; SM90-NEXT: selp.b16 %rs39, %rs17, %rs2, %p9;
+; SM90-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs40;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p17, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p17 bra $L__BB195_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB196_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: add.s16 %rs3, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM90-NEXT: setp.gt.u16 %p2, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2;
+; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p3 bra $L__BB196_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<11>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: $L__BB197_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.gt.u16 %p3, %rs1, %rs5;
+; SM90-NEXT: setp.gt.u16 %p4, %rs2, %rs6;
+; SM90-NEXT: selp.b16 %rs7, %rs6, %rs4, %p4;
+; SM90-NEXT: selp.b16 %rs8, %rs6, %rs7, %p2;
+; SM90-NEXT: selp.b16 %rs9, %rs5, %rs3, %p3;
+; SM90-NEXT: selp.b16 %rs10, %rs5, %rs9, %p1;
+; SM90-NEXT: mov.b32 %r3, {%rs10, %rs8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p5 bra $L__BB197_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<21>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r1;
+; SM90-NEXT: $L__BB198_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r6;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r5;
+; SM90-NEXT: add.s16 %rs7, %rs5, -1;
+; SM90-NEXT: add.s16 %rs8, %rs6, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM90-NEXT: setp.gt.u16 %p5, %rs1, %rs9;
+; SM90-NEXT: setp.gt.u16 %p6, %rs2, %rs10;
+; SM90-NEXT: setp.gt.u16 %p7, %rs5, %rs11;
+; SM90-NEXT: setp.gt.u16 %p8, %rs6, %rs12;
+; SM90-NEXT: selp.b16 %rs13, %rs12, %rs8, %p8;
+; SM90-NEXT: selp.b16 %rs14, %rs12, %rs13, %p4;
+; SM90-NEXT: selp.b16 %rs15, %rs11, %rs7, %p7;
+; SM90-NEXT: selp.b16 %rs16, %rs11, %rs15, %p3;
+; SM90-NEXT: mov.b32 %r3, {%rs16, %rs14};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: selp.b16 %rs17, %rs10, %rs4, %p6;
+; SM90-NEXT: selp.b16 %rs18, %rs10, %rs17, %p2;
+; SM90-NEXT: selp.b16 %rs19, %rs9, %rs3, %p5;
+; SM90-NEXT: selp.b16 %rs20, %rs9, %rs19, %p1;
+; SM90-NEXT: mov.b32 %r4, {%rs20, %rs18};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB198_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<18>;
+; SM90-NEXT: .reg .b16 %rs<41>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs17, %rs18}, %r4;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r3;
+; SM90-NEXT: $L__BB199_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r12;
+; SM90-NEXT: add.s16 %rs3, %rs1, -1;
+; SM90-NEXT: add.s16 %rs4, %rs2, -1;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: add.s16 %rs7, %rs5, -1;
+; SM90-NEXT: add.s16 %rs8, %rs6, -1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r10;
+; SM90-NEXT: add.s16 %rs11, %rs9, -1;
+; SM90-NEXT: add.s16 %rs12, %rs10, -1;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r9;
+; SM90-NEXT: add.s16 %rs15, %rs13, -1;
+; SM90-NEXT: add.s16 %rs16, %rs14, -1;
+; SM90-NEXT: setp.eq.b16 %p1, %rs1, 0;
+; SM90-NEXT: setp.eq.b16 %p2, %rs2, 0;
+; SM90-NEXT: setp.eq.b16 %p3, %rs5, 0;
+; SM90-NEXT: setp.eq.b16 %p4, %rs6, 0;
+; SM90-NEXT: setp.eq.b16 %p5, %rs9, 0;
+; SM90-NEXT: setp.eq.b16 %p6, %rs10, 0;
+; SM90-NEXT: setp.eq.b16 %p7, %rs13, 0;
+; SM90-NEXT: setp.eq.b16 %p8, %rs14, 0;
+; SM90-NEXT: setp.gt.u16 %p9, %rs1, %rs17;
+; SM90-NEXT: setp.gt.u16 %p10, %rs2, %rs18;
+; SM90-NEXT: setp.gt.u16 %p11, %rs5, %rs19;
+; SM90-NEXT: setp.gt.u16 %p12, %rs6, %rs20;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r2;
+; SM90-NEXT: setp.gt.u16 %p13, %rs9, %rs21;
+; SM90-NEXT: setp.gt.u16 %p14, %rs10, %rs22;
+; SM90-NEXT: mov.b32 {%rs23, %rs24}, %r1;
+; SM90-NEXT: setp.gt.u16 %p15, %rs13, %rs23;
+; SM90-NEXT: setp.gt.u16 %p16, %rs14, %rs24;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p16;
+; SM90-NEXT: selp.b16 %rs26, %rs24, %rs25, %p8;
+; SM90-NEXT: selp.b16 %rs27, %rs23, %rs15, %p15;
+; SM90-NEXT: selp.b16 %rs28, %rs23, %rs27, %p7;
+; SM90-NEXT: mov.b32 %r5, {%rs28, %rs26};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: selp.b16 %rs29, %rs22, %rs12, %p14;
+; SM90-NEXT: selp.b16 %rs30, %rs22, %rs29, %p6;
+; SM90-NEXT: selp.b16 %rs31, %rs21, %rs11, %p13;
+; SM90-NEXT: selp.b16 %rs32, %rs21, %rs31, %p5;
+; SM90-NEXT: mov.b32 %r6, {%rs32, %rs30};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b16 %rs33, %rs20, %rs8, %p12;
+; SM90-NEXT: selp.b16 %rs34, %rs20, %rs33, %p4;
+; SM90-NEXT: selp.b16 %rs35, %rs19, %rs7, %p11;
+; SM90-NEXT: selp.b16 %rs36, %rs19, %rs35, %p3;
+; SM90-NEXT: mov.b32 %r7, {%rs36, %rs34};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: selp.b16 %rs37, %rs18, %rs4, %p10;
+; SM90-NEXT: selp.b16 %rs38, %rs18, %rs37, %p2;
+; SM90-NEXT: selp.b16 %rs39, %rs17, %rs3, %p9;
+; SM90-NEXT: selp.b16 %rs40, %rs17, %rs39, %p1;
+; SM90-NEXT: mov.b32 %r8, {%rs40, %rs38};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p17, %rd25, 0;
+; SM90-NEXT: @%p17 bra $L__BB199_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<8>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
+; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd4, %rd7, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM90-NEXT: setp.gt.u64 %p2, %rd7, %rd1;
+; SM90-NEXT: selp.b64 %rd5, %rd1, %rd4, %p2;
+; SM90-NEXT: selp.b64 %rd6, %rd1, %rd5, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
+; SM90-NEXT: setp.ne.b64 %p3, %rd2, %rd7;
+; SM90-NEXT: mov.b64 %rd7, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB204_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd16, %rd17}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB205_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd7, %rd17, -1;
+; SM90-NEXT: add.s64 %rd8, %rd16, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd17, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd16, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd17, %rd5;
+; SM90-NEXT: setp.gt.u64 %p4, %rd16, %rd4;
+; SM90-NEXT: selp.b64 %rd9, %rd4, %rd8, %p4;
+; SM90-NEXT: selp.b64 %rd10, %rd4, %rd9, %p2;
+; SM90-NEXT: selp.b64 %rd11, %rd5, %rd7, %p3;
+; SM90-NEXT: selp.b64 %rd12, %rd5, %rd11, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd17};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd2, %rd17;
+; SM90-NEXT: xor.b64 %rd14, %rd1, %rd16;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p5, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd16, %rd1;
+; SM90-NEXT: mov.b64 %rd17, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB205_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<11>;
+; SM90-NEXT: .reg .b64 %rd<34>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd30, %rd31}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd11, %rd31, -1;
+; SM90-NEXT: add.s64 %rd12, %rd30, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd31, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd30, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd31, %rd2;
+; SM90-NEXT: setp.gt.u64 %p4, %rd30, %rd1;
+; SM90-NEXT: selp.b64 %rd13, %rd1, %rd12, %p4;
+; SM90-NEXT: selp.b64 %rd14, %rd1, %rd13, %p2;
+; SM90-NEXT: selp.b64 %rd15, %rd2, %rd11, %p3;
+; SM90-NEXT: selp.b64 %rd16, %rd2, %rd15, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd31};
+; SM90-NEXT: mov.b128 swap, {%rd14, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd6, %rd31;
+; SM90-NEXT: xor.b64 %rd18, %rd5, %rd30;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p5, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd30, %rd5;
+; SM90-NEXT: mov.b64 %rd31, %rd6;
+; SM90-NEXT: @%p5 bra $L__BB206_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd20, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd32, %rd33}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB206_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd21, %rd33, -1;
+; SM90-NEXT: add.s64 %rd22, %rd32, -1;
+; SM90-NEXT: setp.eq.b64 %p6, %rd33, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd32, 0;
+; SM90-NEXT: setp.gt.u64 %p8, %rd33, %rd4;
+; SM90-NEXT: setp.gt.u64 %p9, %rd32, %rd3;
+; SM90-NEXT: selp.b64 %rd23, %rd3, %rd22, %p9;
+; SM90-NEXT: selp.b64 %rd24, %rd3, %rd23, %p7;
+; SM90-NEXT: selp.b64 %rd25, %rd4, %rd21, %p8;
+; SM90-NEXT: selp.b64 %rd26, %rd4, %rd25, %p6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd33};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd8, %rd33;
+; SM90-NEXT: xor.b64 %rd28, %rd7, %rd32;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p10, %rd29, 0;
+; SM90-NEXT: mov.b64 %rd32, %rd7;
+; SM90-NEXT: mov.b64 %rd33, %rd8;
+; SM90-NEXT: @%p10 bra $L__BB206_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<21>;
+; SM90-NEXT: .reg .b64 %rd<66>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd58, %rd59}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB207_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd59;
+; SM90-NEXT: mov.b64 %rd9, %rd58;
+; SM90-NEXT: add.s64 %rd19, %rd10, -1;
+; SM90-NEXT: add.s64 %rd20, %rd9, -1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd10, 0;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: setp.gt.u64 %p3, %rd10, %rd6;
+; SM90-NEXT: setp.gt.u64 %p4, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, %rd5, %rd20, %p4;
+; SM90-NEXT: selp.b64 %rd22, %rd5, %rd21, %p2;
+; SM90-NEXT: selp.b64 %rd23, %rd6, %rd19, %p3;
+; SM90-NEXT: selp.b64 %rd24, %rd6, %rd23, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd58, %rd59}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd59, %rd10;
+; SM90-NEXT: xor.b64 %rd26, %rd58, %rd9;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: setp.ne.b64 %p5, %rd27, 0;
+; SM90-NEXT: @%p5 bra $L__BB207_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd60, %rd61}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB207_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd61;
+; SM90-NEXT: mov.b64 %rd11, %rd60;
+; SM90-NEXT: add.s64 %rd29, %rd12, -1;
+; SM90-NEXT: add.s64 %rd30, %rd11, -1;
+; SM90-NEXT: setp.eq.b64 %p6, %rd12, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd11, 0;
+; SM90-NEXT: setp.gt.u64 %p8, %rd12, %rd8;
+; SM90-NEXT: setp.gt.u64 %p9, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd31, %rd7, %rd30, %p9;
+; SM90-NEXT: selp.b64 %rd32, %rd7, %rd31, %p7;
+; SM90-NEXT: selp.b64 %rd33, %rd8, %rd29, %p8;
+; SM90-NEXT: selp.b64 %rd34, %rd8, %rd33, %p6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd60, %rd61}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd61, %rd12;
+; SM90-NEXT: xor.b64 %rd36, %rd60, %rd11;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p10, %rd37, 0;
+; SM90-NEXT: @%p10 bra $L__BB207_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd38, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd38, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd62, %rd63}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB207_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd39, %rd63, -1;
+; SM90-NEXT: add.s64 %rd40, %rd62, -1;
+; SM90-NEXT: setp.eq.b64 %p11, %rd63, 0;
+; SM90-NEXT: setp.eq.b64 %p12, %rd62, 0;
+; SM90-NEXT: setp.gt.u64 %p13, %rd63, %rd2;
+; SM90-NEXT: setp.gt.u64 %p14, %rd62, %rd1;
+; SM90-NEXT: selp.b64 %rd41, %rd1, %rd40, %p14;
+; SM90-NEXT: selp.b64 %rd42, %rd1, %rd41, %p12;
+; SM90-NEXT: selp.b64 %rd43, %rd2, %rd39, %p13;
+; SM90-NEXT: selp.b64 %rd44, %rd2, %rd43, %p11;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd62, %rd63};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd44};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd45, %rd14, %rd63;
+; SM90-NEXT: xor.b64 %rd46, %rd13, %rd62;
+; SM90-NEXT: or.b64 %rd47, %rd46, %rd45;
+; SM90-NEXT: setp.ne.b64 %p15, %rd47, 0;
+; SM90-NEXT: mov.b64 %rd62, %rd13;
+; SM90-NEXT: mov.b64 %rd63, %rd14;
+; SM90-NEXT: @%p15 bra $L__BB207_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd48, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd48};
+; SM90-NEXT: mov.b128 swap, {%rd48, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd64, %rd65}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB207_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s64 %rd49, %rd65, -1;
+; SM90-NEXT: add.s64 %rd50, %rd64, -1;
+; SM90-NEXT: setp.eq.b64 %p16, %rd65, 0;
+; SM90-NEXT: setp.eq.b64 %p17, %rd64, 0;
+; SM90-NEXT: setp.gt.u64 %p18, %rd65, %rd4;
+; SM90-NEXT: setp.gt.u64 %p19, %rd64, %rd3;
+; SM90-NEXT: selp.b64 %rd51, %rd3, %rd50, %p19;
+; SM90-NEXT: selp.b64 %rd52, %rd3, %rd51, %p17;
+; SM90-NEXT: selp.b64 %rd53, %rd4, %rd49, %p18;
+; SM90-NEXT: selp.b64 %rd54, %rd4, %rd53, %p16;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd64, %rd65};
+; SM90-NEXT: mov.b128 swap, {%rd52, %rd54};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd55, %rd16, %rd65;
+; SM90-NEXT: xor.b64 %rd56, %rd15, %rd64;
+; SM90-NEXT: or.b64 %rd57, %rd56, %rd55;
+; SM90-NEXT: setp.ne.b64 %p20, %rd57, 0;
+; SM90-NEXT: mov.b64 %rd64, %rd15;
+; SM90-NEXT: mov.b64 %rd65, %rd16;
+; SM90-NEXT: @%p20 bra $L__BB207_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB208_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: and.b32 %r10, %r9, 255;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p2 bra $L__BB208_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<15>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: mov.b32 {%rs6, %rs7}, %r4;
+; SM90-NEXT: $L__BB209_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r15, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: and.b16 %rs4, %rs3, 255;
+; SM90-NEXT: shr.u16 %rs5, %rs3, 8;
+; SM90-NEXT: setp.ge.u16 %p1, %rs5, %rs7;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs6;
+; SM90-NEXT: sub.s16 %rs8, %rs5, %rs7;
+; SM90-NEXT: sub.s16 %rs9, %rs3, %rs6;
+; SM90-NEXT: selp.b16 %rs10, %rs9, %rs3, %p2;
+; SM90-NEXT: selp.b16 %rs11, %rs8, %rs5, %p1;
+; SM90-NEXT: shl.b16 %rs12, %rs11, 8;
+; SM90-NEXT: and.b16 %rs13, %rs10, 255;
+; SM90-NEXT: or.b16 %rs14, %rs13, %rs12;
+; SM90-NEXT: cvt.u32.u16 %r10, %rs14;
+; SM90-NEXT: shl.b32 %r11, %r10, %r1;
+; SM90-NEXT: and.b32 %r12, %r15, %r2;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15;
+; SM90-NEXT: mov.b32 %r15, %r3;
+; SM90-NEXT: @%p3 bra $L__BB209_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<23>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r3;
+; SM90-NEXT: $L__BB210_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r22, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r4;
+; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs3;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r8;
+; SM90-NEXT: sub.s16 %rs10, %rs9, %rs2;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r10;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r11, %rs12;
+; SM90-NEXT: selp.b32 %r12, %r11, %r10, %p4;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs10;
+; SM90-NEXT: selp.b32 %r14, %r13, %r8, %p3;
+; SM90-NEXT: prmt.b32 %r15, %r14, %r12, 0x3340U;
+; SM90-NEXT: cvt.u32.u16 %r16, %rs8;
+; SM90-NEXT: selp.b32 %r17, %r16, %r6, %p2;
+; SM90-NEXT: cvt.u32.u16 %r18, %rs6;
+; SM90-NEXT: selp.b32 %r19, %r18, %r4, %p1;
+; SM90-NEXT: prmt.b32 %r20, %r19, %r17, 0x3340U;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r15, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
+; SM90-NEXT: setp.ne.b32 %p5, %r1, %r22;
+; SM90-NEXT: mov.b32 %r22, %r1;
+; SM90-NEXT: @%p5 bra $L__BB210_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r5;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r3;
+; SM90-NEXT: $L__BB211_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r8, %r34, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r10, %r34, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM90-NEXT: prmt.b32 %r11, %r1, 0, 0x7770U;
+; SM90-NEXT: prmt.b32 %r12, %r33, 0, 0x7770U;
+; SM90-NEXT: setp.ge.u32 %p5, %r12, %r11;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7771U;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7771U;
+; SM90-NEXT: setp.ge.u32 %p6, %r14, %r13;
+; SM90-NEXT: prmt.b32 %r15, %r1, 0, 0x7772U;
+; SM90-NEXT: prmt.b32 %r16, %r33, 0, 0x7772U;
+; SM90-NEXT: setp.ge.u32 %p7, %r16, %r15;
+; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
+; SM90-NEXT: setp.ge.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r18;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r15;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r16;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r13;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r14;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r12;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r10;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r7;
+; SM90-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r4;
+; SM90-NEXT: sub.s16 %rs17, %rs16, %rs15;
+; SM90-NEXT: sub.s16 %rs18, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs19, %rs12, %rs11;
+; SM90-NEXT: sub.s16 %rs20, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs21, %rs8, %rs7;
+; SM90-NEXT: sub.s16 %rs22, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs23, %rs4, %rs3;
+; SM90-NEXT: sub.s16 %rs24, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs2, %p8;
+; SM90-NEXT: cvt.u32.u16 %r19, %rs25;
+; SM90-NEXT: selp.b16 %rs26, %rs23, %rs4, %p7;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs26;
+; SM90-NEXT: prmt.b32 %r21, %r20, %r19, 0x3340U;
+; SM90-NEXT: selp.b16 %rs27, %rs22, %rs6, %p6;
+; SM90-NEXT: cvt.u32.u16 %r22, %rs27;
+; SM90-NEXT: selp.b16 %rs28, %rs21, %rs8, %p5;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs28;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r22, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r24, %r21, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r25;
+; SM90-NEXT: selp.b16 %rs29, %rs20, %rs10, %p4;
+; SM90-NEXT: cvt.u32.u16 %r26, %rs29;
+; SM90-NEXT: selp.b16 %rs30, %rs19, %rs12, %p3;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs30;
+; SM90-NEXT: prmt.b32 %r28, %r27, %r26, 0x3340U;
+; SM90-NEXT: selp.b16 %rs31, %rs18, %rs14, %p2;
+; SM90-NEXT: cvt.u32.u16 %r29, %rs31;
+; SM90-NEXT: selp.b16 %rs32, %rs17, %rs16, %p1;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r29, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r28, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r32;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p9, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p9 bra $L__BB211_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB212_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p2 bra $L__BB212_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: $L__BB213_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs5, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs6, %rs4, %rs2;
+; SM90-NEXT: selp.b16 %rs7, %rs6, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs8, %rs5, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r3, {%rs8, %rs7};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p3, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p3 bra $L__BB213_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT: $L__BB214_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r6;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: sub.s16 %rs9, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs10, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs11, %rs7, %rs5;
+; SM90-NEXT: sub.s16 %rs12, %rs8, %rs6;
+; SM90-NEXT: selp.b16 %rs13, %rs12, %rs8, %p4;
+; SM90-NEXT: selp.b16 %rs14, %rs11, %rs7, %p3;
+; SM90-NEXT: mov.b32 %r3, {%rs14, %rs13};
+; SM90-NEXT: cvt.u64.u32 %rd7, %r3;
+; SM90-NEXT: selp.b16 %rs15, %rs10, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs16, %rs9, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r4, {%rs16, %rs15};
+; SM90-NEXT: cvt.u64.u32 %rd8, %r4;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p5, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p5 bra $L__BB214_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT: $L__BB215_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1;
+; SM90-NEXT: setp.ge.u16 %p2, %rs4, %rs2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r11;
+; SM90-NEXT: setp.ge.u16 %p3, %rs7, %rs5;
+; SM90-NEXT: setp.ge.u16 %p4, %rs8, %rs6;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: mov.b32 {%rs11, %rs12}, %r10;
+; SM90-NEXT: setp.ge.u16 %p5, %rs11, %rs9;
+; SM90-NEXT: setp.ge.u16 %p6, %rs12, %rs10;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r1;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r9;
+; SM90-NEXT: setp.ge.u16 %p7, %rs15, %rs13;
+; SM90-NEXT: setp.ge.u16 %p8, %rs16, %rs14;
+; SM90-NEXT: sub.s16 %rs17, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs18, %rs4, %rs2;
+; SM90-NEXT: sub.s16 %rs19, %rs7, %rs5;
+; SM90-NEXT: sub.s16 %rs20, %rs8, %rs6;
+; SM90-NEXT: sub.s16 %rs21, %rs11, %rs9;
+; SM90-NEXT: sub.s16 %rs22, %rs12, %rs10;
+; SM90-NEXT: sub.s16 %rs23, %rs15, %rs13;
+; SM90-NEXT: sub.s16 %rs24, %rs16, %rs14;
+; SM90-NEXT: selp.b16 %rs25, %rs24, %rs16, %p8;
+; SM90-NEXT: selp.b16 %rs26, %rs23, %rs15, %p7;
+; SM90-NEXT: mov.b32 %r5, {%rs26, %rs25};
+; SM90-NEXT: cvt.u64.u32 %rd13, %r5;
+; SM90-NEXT: selp.b16 %rs27, %rs22, %rs12, %p6;
+; SM90-NEXT: selp.b16 %rs28, %rs21, %rs11, %p5;
+; SM90-NEXT: mov.b32 %r6, {%rs28, %rs27};
+; SM90-NEXT: cvt.u64.u32 %rd14, %r6;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b16 %rs29, %rs20, %rs8, %p4;
+; SM90-NEXT: selp.b16 %rs30, %rs19, %rs7, %p3;
+; SM90-NEXT: mov.b32 %r7, {%rs30, %rs29};
+; SM90-NEXT: cvt.u64.u32 %rd17, %r7;
+; SM90-NEXT: selp.b16 %rs31, %rs18, %rs4, %p2;
+; SM90-NEXT: selp.b16 %rs32, %rs17, %rs3, %p1;
+; SM90-NEXT: mov.b32 %r8, {%rs32, %rs31};
+; SM90-NEXT: cvt.u64.u32 %rd18, %r8;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd25, 0;
+; SM90-NEXT: @%p9 bra $L__BB215_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB216_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u32 %p1, %r5, %r1;
+; SM90-NEXT: sub.s32 %r3, %r5, %r1;
+; SM90-NEXT: selp.b32 %r4, %r3, %r5, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p2, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p2 bra $L__BB216_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB217_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: setp.ge.u32 %p1, %r8, %r2;
+; SM90-NEXT: setp.ge.u32 %p2, %r7, %r1;
+; SM90-NEXT: sub.s32 %r3, %r8, %r2;
+; SM90-NEXT: sub.s32 %r4, %r7, %r1;
+; SM90-NEXT: selp.b32 %r5, %r4, %r7, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: selp.b32 %r6, %r3, %r8, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p3, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p3 bra $L__BB217_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB218_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r14;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: setp.ge.u32 %p1, %r16, %r4;
+; SM90-NEXT: setp.ge.u32 %p2, %r15, %r3;
+; SM90-NEXT: setp.ge.u32 %p3, %r14, %r2;
+; SM90-NEXT: setp.ge.u32 %p4, %r13, %r1;
+; SM90-NEXT: sub.s32 %r5, %r16, %r4;
+; SM90-NEXT: sub.s32 %r6, %r15, %r3;
+; SM90-NEXT: sub.s32 %r7, %r14, %r2;
+; SM90-NEXT: sub.s32 %r8, %r13, %r1;
+; SM90-NEXT: selp.b32 %r9, %r8, %r13, %p4;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: selp.b32 %r10, %r7, %r14, %p3;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b32 %r11, %r6, %r15, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: selp.b32 %r12, %r5, %r16, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
+; SM90-NEXT: @%p5 bra $L__BB218_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<11>;
+; SM90-NEXT: .reg .b32 %r<33>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r28}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
+; SM90-NEXT: $L__BB219_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r25;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r26;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r27;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r28;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: setp.ge.u32 %p1, %r28, %r4;
+; SM90-NEXT: setp.ge.u32 %p2, %r27, %r3;
+; SM90-NEXT: setp.ge.u32 %p3, %r26, %r2;
+; SM90-NEXT: setp.ge.u32 %p4, %r25, %r1;
+; SM90-NEXT: sub.s32 %r9, %r28, %r4;
+; SM90-NEXT: sub.s32 %r10, %r27, %r3;
+; SM90-NEXT: sub.s32 %r11, %r26, %r2;
+; SM90-NEXT: sub.s32 %r12, %r25, %r1;
+; SM90-NEXT: selp.b32 %r13, %r12, %r25, %p4;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: selp.b32 %r14, %r11, %r26, %p3;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: selp.b32 %r15, %r10, %r27, %p2;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: selp.b32 %r16, %r9, %r28, %p1;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd12;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd8;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r28}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd22;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
+; SM90-NEXT: setp.ne.b64 %p5, %rd25, 0;
+; SM90-NEXT: @%p5 bra $L__BB219_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r32}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
+; SM90-NEXT: $L__BB219_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd29, %r29;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r30;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r31;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r32;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: setp.ge.u32 %p6, %r32, %r8;
+; SM90-NEXT: setp.ge.u32 %p7, %r31, %r7;
+; SM90-NEXT: setp.ge.u32 %p8, %r30, %r6;
+; SM90-NEXT: setp.ge.u32 %p9, %r29, %r5;
+; SM90-NEXT: sub.s32 %r17, %r32, %r8;
+; SM90-NEXT: sub.s32 %r18, %r31, %r7;
+; SM90-NEXT: sub.s32 %r19, %r30, %r6;
+; SM90-NEXT: sub.s32 %r20, %r29, %r5;
+; SM90-NEXT: selp.b32 %r21, %r20, %r29, %p9;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: selp.b32 %r22, %r19, %r30, %p8;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: selp.b32 %r23, %r18, %r31, %p7;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: selp.b32 %r24, %r17, %r32, %p6;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd40, %rd44};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd36;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd32;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r32}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd46;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
+; SM90-NEXT: setp.ne.b64 %p10, %rd49, 0;
+; SM90-NEXT: @%p10 bra $L__BB219_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd1;
+; SM90-NEXT: sub.s64 %rd4, %rd6, %rd1;
+; SM90-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p2, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p2 bra $L__BB220_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<4>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB221_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd15, %rd5;
+; SM90-NEXT: setp.ge.u64 %p2, %rd14, %rd4;
+; SM90-NEXT: sub.s64 %rd7, %rd15, %rd5;
+; SM90-NEXT: sub.s64 %rd8, %rd14, %rd4;
+; SM90-NEXT: selp.b64 %rd9, %rd8, %rd14, %p2;
+; SM90-NEXT: selp.b64 %rd10, %rd7, %rd15, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd9, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p3, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p3 bra $L__BB221_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<7>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB222_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p1, %rd27, %rd2;
+; SM90-NEXT: setp.ge.u64 %p2, %rd26, %rd1;
+; SM90-NEXT: sub.s64 %rd11, %rd27, %rd2;
+; SM90-NEXT: sub.s64 %rd12, %rd26, %rd1;
+; SM90-NEXT: selp.b64 %rd13, %rd12, %rd26, %p2;
+; SM90-NEXT: selp.b64 %rd14, %rd11, %rd27, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd13, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p3, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p3 bra $L__BB222_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB222_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p4, %rd29, %rd4;
+; SM90-NEXT: setp.ge.u64 %p5, %rd28, %rd3;
+; SM90-NEXT: sub.s64 %rd19, %rd29, %rd4;
+; SM90-NEXT: sub.s64 %rd20, %rd28, %rd3;
+; SM90-NEXT: selp.b64 %rd21, %rd20, %rd28, %p5;
+; SM90-NEXT: selp.b64 %rd22, %rd19, %rd29, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p6, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p6 bra $L__BB222_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: usub_cond_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<13>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB223_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: setp.ge.u64 %p1, %rd10, %rd6;
+; SM90-NEXT: setp.ge.u64 %p2, %rd9, %rd5;
+; SM90-NEXT: sub.s64 %rd19, %rd10, %rd6;
+; SM90-NEXT: sub.s64 %rd20, %rd9, %rd5;
+; SM90-NEXT: selp.b64 %rd21, %rd20, %rd9, %p2;
+; SM90-NEXT: selp.b64 %rd22, %rd19, %rd10, %p1;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd21, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p3, %rd25, 0;
+; SM90-NEXT: @%p3 bra $L__BB223_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB223_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: setp.ge.u64 %p4, %rd12, %rd8;
+; SM90-NEXT: setp.ge.u64 %p5, %rd11, %rd7;
+; SM90-NEXT: sub.s64 %rd27, %rd12, %rd8;
+; SM90-NEXT: sub.s64 %rd28, %rd11, %rd7;
+; SM90-NEXT: selp.b64 %rd29, %rd28, %rd11, %p5;
+; SM90-NEXT: selp.b64 %rd30, %rd27, %rd12, %p4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p6, %rd33, 0;
+; SM90-NEXT: @%p6 bra $L__BB223_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB223_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p7, %rd55, %rd2;
+; SM90-NEXT: setp.ge.u64 %p8, %rd54, %rd1;
+; SM90-NEXT: sub.s64 %rd35, %rd55, %rd2;
+; SM90-NEXT: sub.s64 %rd36, %rd54, %rd1;
+; SM90-NEXT: selp.b64 %rd37, %rd36, %rd54, %p8;
+; SM90-NEXT: selp.b64 %rd38, %rd35, %rd55, %p7;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p9, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p9 bra $L__BB223_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB223_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.ge.u64 %p10, %rd57, %rd4;
+; SM90-NEXT: setp.ge.u64 %p11, %rd56, %rd3;
+; SM90-NEXT: sub.s64 %rd43, %rd57, %rd4;
+; SM90-NEXT: sub.s64 %rd44, %rd56, %rd3;
+; SM90-NEXT: selp.b64 %rd45, %rd44, %rd56, %p11;
+; SM90-NEXT: selp.b64 %rd46, %rd43, %rd57, %p10;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd45, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p12, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p12 bra $L__BB223_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<6>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 255;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB224_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: and.b16 %rs3, %rs2, 255;
+; SM90-NEXT: max.u16 %rs4, %rs3, %rs1;
+; SM90-NEXT: sub.s16 %rs5, %rs4, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs5;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB224_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT: $L__BB225_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, 16711935;
+; SM90-NEXT: max.u16x2 %r12, %r11, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: sub.s16 %rs9, %rs5, %rs7;
+; SM90-NEXT: sub.s16 %rs10, %rs6, %rs8;
+; SM90-NEXT: shl.b16 %rs11, %rs10, 8;
+; SM90-NEXT: or.b16 %rs12, %rs9, %rs11;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs12;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB225_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM90-NEXT: $L__BB226_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM90-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM90-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM90-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB226_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<33>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB227_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs6, %r7;
+; SM90-NEXT: max.u16 %rs7, %rs6, %rs5;
+; SM90-NEXT: sub.s16 %rs8, %rs7, %rs5;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs8;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r11;
+; SM90-NEXT: max.u16 %rs11, %rs10, %rs9;
+; SM90-NEXT: sub.s16 %rs12, %rs11, %rs9;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r14;
+; SM90-NEXT: max.u16 %rs15, %rs14, %rs13;
+; SM90-NEXT: sub.s16 %rs16, %rs15, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs16;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs18, %r19;
+; SM90-NEXT: max.u16 %rs19, %rs18, %rs17;
+; SM90-NEXT: sub.s16 %rs20, %rs19, %rs17;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM90-NEXT: cvt.u16.u32 %rs21, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r22;
+; SM90-NEXT: max.u16 %rs23, %rs22, %rs21;
+; SM90-NEXT: sub.s16 %rs24, %rs23, %rs21;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs24;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs25, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs26, %r26;
+; SM90-NEXT: max.u16 %rs27, %rs26, %rs25;
+; SM90-NEXT: sub.s16 %rs28, %rs27, %rs25;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs28;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs29, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs30, %r29;
+; SM90-NEXT: max.u16 %rs31, %rs30, %rs29;
+; SM90-NEXT: sub.s16 %rs32, %rs31, %rs29;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs32;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB227_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
+ ret <8 x i8> %retval
+}
+
+define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB228_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
+; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs4;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB228_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
+ ret <1 x i16> %retval
+}
+
+define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<7>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r2;
+; SM90-NEXT: $L__BB229_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r5, %r2;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r4, {%rs6, %rs5};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB229_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
+ ret <2 x i16> %retval
+}
+
+define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: $L__BB230_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r3, %r7, %r1;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r4, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM90-NEXT: max.u16x2 %r5, %r8, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r5;
+; SM90-NEXT: sub.s16 %rs11, %rs8, %rs10;
+; SM90-NEXT: sub.s16 %rs12, %rs7, %rs9;
+; SM90-NEXT: mov.b32 %r6, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB230_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
+ ret <4 x i16> %retval
+}
+
+define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT: mov.b32 {%rs9, %rs10}, %r2;
+; SM90-NEXT: $L__BB231_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u16x2 %r5, %r13, %r1;
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r5;
+; SM90-NEXT: sub.s16 %rs5, %rs2, %rs4;
+; SM90-NEXT: sub.s16 %rs6, %rs1, %rs3;
+; SM90-NEXT: mov.b32 %r6, {%rs6, %rs5};
+; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
+; SM90-NEXT: max.u16x2 %r7, %r14, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r7;
+; SM90-NEXT: sub.s16 %rs11, %rs8, %rs10;
+; SM90-NEXT: sub.s16 %rs12, %rs7, %rs9;
+; SM90-NEXT: mov.b32 %r8, {%rs12, %rs11};
+; SM90-NEXT: cvt.u64.u32 %rd6, %r8;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u16x2 %r9, %r15, %r3;
+; SM90-NEXT: mov.b32 {%rs13, %rs14}, %r9;
+; SM90-NEXT: mov.b32 {%rs15, %rs16}, %r3;
+; SM90-NEXT: sub.s16 %rs17, %rs14, %rs16;
+; SM90-NEXT: sub.s16 %rs18, %rs13, %rs15;
+; SM90-NEXT: mov.b32 %r10, {%rs18, %rs17};
+; SM90-NEXT: cvt.u64.u32 %rd9, %r10;
+; SM90-NEXT: max.u16x2 %r11, %r16, %r4;
+; SM90-NEXT: mov.b32 {%rs19, %rs20}, %r11;
+; SM90-NEXT: mov.b32 {%rs21, %rs22}, %r4;
+; SM90-NEXT: sub.s16 %rs23, %rs20, %rs22;
+; SM90-NEXT: sub.s16 %rs24, %rs19, %rs21;
+; SM90-NEXT: mov.b32 %r12, {%rs24, %rs23};
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB231_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
+ ret <8 x i16> %retval
+}
+
+define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r3, %r5, %r1;
+; SM90-NEXT: sub.s32 %r4, %r3, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB232_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB233_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r3, %r7, %r1;
+; SM90-NEXT: sub.s32 %r4, %r3, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r4;
+; SM90-NEXT: max.u32 %r5, %r8, %r2;
+; SM90-NEXT: sub.s32 %r6, %r5, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r8;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB233_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r16}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd4;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd3;
+; SM90-NEXT: $L__BB234_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r5, %r13, %r1;
+; SM90-NEXT: sub.s32 %r6, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r6;
+; SM90-NEXT: max.u32 %r7, %r14, %r2;
+; SM90-NEXT: sub.s32 %r8, %r7, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r8;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u32 %r9, %r15, %r3;
+; SM90-NEXT: sub.s32 %r10, %r9, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r10;
+; SM90-NEXT: max.u32 %r11, %r16, %r4;
+; SM90-NEXT: sub.s32 %r12, %r11, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r13;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r14;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r15;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r16;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r16}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r15, %rd22;
+; SM90-NEXT: mov.b64 {_, %r14}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r13, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB234_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<33>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r28}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd4;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd3;
+; SM90-NEXT: $L__BB235_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r9, %r25, %r1;
+; SM90-NEXT: sub.s32 %r10, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r10;
+; SM90-NEXT: max.u32 %r11, %r26, %r2;
+; SM90-NEXT: sub.s32 %r12, %r11, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r12;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.u32 %r13, %r27, %r3;
+; SM90-NEXT: sub.s32 %r14, %r13, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r14;
+; SM90-NEXT: max.u32 %r15, %r28, %r4;
+; SM90-NEXT: sub.s32 %r16, %r15, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r16;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r26;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r27;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r28}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r27, %rd22;
+; SM90-NEXT: mov.b64 {_, %r26}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r25, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB235_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r32}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd28;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd27;
+; SM90-NEXT: $L__BB235_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u32 %r17, %r29, %r5;
+; SM90-NEXT: sub.s32 %r18, %r17, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r18;
+; SM90-NEXT: max.u32 %r19, %r30, %r6;
+; SM90-NEXT: sub.s32 %r20, %r19, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r20;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.u32 %r21, %r31, %r7;
+; SM90-NEXT: sub.s32 %r22, %r21, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r22;
+; SM90-NEXT: max.u32 %r23, %r32, %r8;
+; SM90-NEXT: sub.s32 %r24, %r23, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r24;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r29;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r30;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r31;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r32;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r32}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r31, %rd46;
+; SM90-NEXT: mov.b64 {_, %r30}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r29, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB235_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
+ ret <8 x i32> %retval
+}
+
+define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v1i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<7>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
+; SM90-NEXT: $L__BB236_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd4, %rd6, %rd1;
+; SM90-NEXT: sub.s64 %rd5, %rd4, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd6;
+; SM90-NEXT: mov.b64 %rd6, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB236_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
+ ret <1 x i64> %retval
+}
+
+define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v2i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<16>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd14, %rd15}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB237_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd7, %rd14, %rd4;
+; SM90-NEXT: sub.s64 %rd8, %rd7, %rd4;
+; SM90-NEXT: max.u64 %rd9, %rd15, %rd5;
+; SM90-NEXT: sub.s64 %rd10, %rd9, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd14, %rd15};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd11, %rd2, %rd15;
+; SM90-NEXT: xor.b64 %rd12, %rd1, %rd14;
+; SM90-NEXT: or.b64 %rd13, %rd12, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd13, 0;
+; SM90-NEXT: mov.b64 %rd14, %rd1;
+; SM90-NEXT: mov.b64 %rd15, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB237_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
+ ret <2 x i64> %retval
+}
+
+define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v4i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<30>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd26, %rd27}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB238_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd11, %rd26, %rd1;
+; SM90-NEXT: sub.s64 %rd12, %rd11, %rd1;
+; SM90-NEXT: max.u64 %rd13, %rd27, %rd2;
+; SM90-NEXT: sub.s64 %rd14, %rd13, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd27};
+; SM90-NEXT: mov.b128 swap, {%rd12, %rd14};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd15, %rd6, %rd27;
+; SM90-NEXT: xor.b64 %rd16, %rd5, %rd26;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p1, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd26, %rd5;
+; SM90-NEXT: mov.b64 %rd27, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB238_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd28, %rd29}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB238_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd19, %rd28, %rd3;
+; SM90-NEXT: sub.s64 %rd20, %rd19, %rd3;
+; SM90-NEXT: max.u64 %rd21, %rd29, %rd4;
+; SM90-NEXT: sub.s64 %rd22, %rd21, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd29};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd8, %rd29;
+; SM90-NEXT: xor.b64 %rd24, %rd7, %rd28;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p2, %rd25, 0;
+; SM90-NEXT: mov.b64 %rd28, %rd7;
+; SM90-NEXT: mov.b64 %rd29, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB238_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
+ ret <4 x i64> %retval
+}
+
+define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
+; SM90-LABEL: usub_sat_acq_rel_v8i64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<58>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB239_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd51;
+; SM90-NEXT: mov.b64 %rd9, %rd50;
+; SM90-NEXT: max.u64 %rd19, %rd9, %rd5;
+; SM90-NEXT: sub.s64 %rd20, %rd19, %rd5;
+; SM90-NEXT: max.u64 %rd21, %rd10, %rd6;
+; SM90-NEXT: sub.s64 %rd22, %rd21, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd20, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd50, %rd51}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd51, %rd10;
+; SM90-NEXT: xor.b64 %rd24, %rd50, %rd9;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB239_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB239_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd53;
+; SM90-NEXT: mov.b64 %rd11, %rd52;
+; SM90-NEXT: max.u64 %rd27, %rd11, %rd7;
+; SM90-NEXT: sub.s64 %rd28, %rd27, %rd7;
+; SM90-NEXT: max.u64 %rd29, %rd12, %rd8;
+; SM90-NEXT: sub.s64 %rd30, %rd29, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd52, %rd53}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd31, %rd53, %rd12;
+; SM90-NEXT: xor.b64 %rd32, %rd52, %rd11;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p2, %rd33, 0;
+; SM90-NEXT: @%p2 bra $L__BB239_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd34, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd34};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd54, %rd55}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB239_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd35, %rd54, %rd1;
+; SM90-NEXT: sub.s64 %rd36, %rd35, %rd1;
+; SM90-NEXT: max.u64 %rd37, %rd55, %rd2;
+; SM90-NEXT: sub.s64 %rd38, %rd37, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd55};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd14, %rd55;
+; SM90-NEXT: xor.b64 %rd40, %rd13, %rd54;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p3, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd54, %rd13;
+; SM90-NEXT: mov.b64 %rd55, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB239_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd42, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd56, %rd57}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB239_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.u64 %rd43, %rd56, %rd3;
+; SM90-NEXT: sub.s64 %rd44, %rd43, %rd3;
+; SM90-NEXT: max.u64 %rd45, %rd57, %rd4;
+; SM90-NEXT: sub.s64 %rd46, %rd45, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd56, %rd57};
+; SM90-NEXT: mov.b128 swap, {%rd44, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd16, %rd57;
+; SM90-NEXT: xor.b64 %rd48, %rd15, %rd56;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: setp.ne.b64 %p4, %rd49, 0;
+; SM90-NEXT: mov.b64 %rd56, %rd15;
+; SM90-NEXT: mov.b64 %rd57, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB239_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
+ ret <8 x i64> %retval
+}
+
+define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB244_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB244_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB245_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB245_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB246_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB246_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB247_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: sub.rn.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: sub.rn.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB247_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB247_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: sub.rn.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: sub.rn.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: sub.rn.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB247_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB248_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB248_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB249_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB249_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB250_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB250_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB251_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: min.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: min.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB251_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB251_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: min.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: min.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: min.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB251_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB252_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB252_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB253_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB253_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB254_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB254_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB255_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: max.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: max.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB255_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB255_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: max.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: max.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB255_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB256_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB256_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB257_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB257_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB258_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB258_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB259_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: min.NaN.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: min.NaN.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB259_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB259_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: min.NaN.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: min.NaN.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: min.NaN.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB259_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB260_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r3, %r4, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r4;
+; SM90-NEXT: mov.b32 %r4, %r2;
+; SM90-NEXT: @%p1 bra $L__BB260_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ ret <1 x float> %retval
+}
+
+define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB261_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.f32 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB261_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ ret <2 x float> %retval
+}
+
+define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB262_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.f32 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f32 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.f32 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB262_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ ret <4 x float> %retval
+}
+
+define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB263_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: max.NaN.f32 %r10, %r18, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f32 %r11, %r19, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: max.NaN.f32 %r12, %r20, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r18;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r20;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd22;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB263_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd26, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd26, %rd26};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd27, %rd28}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd28;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd28;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd27;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd27;
+; SM90-NEXT: $L__BB263_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f32 %r13, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd29, %r13;
+; SM90-NEXT: max.NaN.f32 %r14, %r22, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd30, %r14;
+; SM90-NEXT: shl.b64 %rd31, %rd30, 32;
+; SM90-NEXT: or.b64 %rd32, %rd29, %rd31;
+; SM90-NEXT: max.NaN.f32 %r15, %r23, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd33, %r15;
+; SM90-NEXT: max.NaN.f32 %r16, %r24, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd34, %r16;
+; SM90-NEXT: shl.b64 %rd35, %rd34, 32;
+; SM90-NEXT: or.b64 %rd36, %rd33, %rd35;
+; SM90-NEXT: cvt.u64.u32 %rd37, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd38, %r22;
+; SM90-NEXT: shl.b64 %rd39, %rd38, 32;
+; SM90-NEXT: or.b64 %rd40, %rd37, %rd39;
+; SM90-NEXT: cvt.u64.u32 %rd41, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd42, %r24;
+; SM90-NEXT: shl.b64 %rd43, %rd42, 32;
+; SM90-NEXT: or.b64 %rd44, %rd41, %rd43;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd44};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd45, %rd46}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd47, %rd46, %rd44;
+; SM90-NEXT: xor.b64 %rd48, %rd45, %rd40;
+; SM90-NEXT: or.b64 %rd49, %rd48, %rd47;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd46;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd46;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd45;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd45;
+; SM90-NEXT: setp.ne.b64 %p2, %rd49, 0;
+; SM90-NEXT: @%p2 bra $L__BB263_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ ret <8 x float> %retval
+}
+
+define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<4>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b64 %rd<18>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
+; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB268_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB268_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB269_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: sub.rn.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB269_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB270_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: sub.rn.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB270_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB270_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: sub.rn.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB270_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB271_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: sub.rn.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: sub.rn.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB271_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB271_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: sub.rn.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: sub.rn.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB271_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB271_5: // %atomicrmw.start16
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: sub.rn.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB271_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end15
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB271_7: // %atomicrmw.start22
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: sub.rn.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB271_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end21
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB272_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB272_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB273_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: min.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB273_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB274_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: min.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB274_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB274_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: min.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB274_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB275_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: min.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: min.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB275_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB275_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: min.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: min.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB275_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB275_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: min.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB275_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB275_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: min.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB275_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<6>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
+; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd4, %rd5, %rd1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
+; SM90-NEXT: setp.ne.b64 %p1, %rd2, %rd5;
+; SM90-NEXT: mov.b64 %rd5, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB276_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b64 %rd<14>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd12, %rd13}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB277_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd7, %rd12, %rd4;
+; SM90-NEXT: max.f64 %rd8, %rd13, %rd5;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd12, %rd13};
+; SM90-NEXT: mov.b128 swap, {%rd7, %rd8};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd9, %rd2, %rd13;
+; SM90-NEXT: xor.b64 %rd10, %rd1, %rd12;
+; SM90-NEXT: or.b64 %rd11, %rd10, %rd9;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, 0;
+; SM90-NEXT: mov.b64 %rd12, %rd1;
+; SM90-NEXT: mov.b64 %rd13, %rd2;
+; SM90-NEXT: @%p1 bra $L__BB277_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd22, %rd23}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB278_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd11, %rd22, %rd1;
+; SM90-NEXT: max.f64 %rd12, %rd23, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd23};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd13, %rd6, %rd23;
+; SM90-NEXT: xor.b64 %rd14, %rd5, %rd22;
+; SM90-NEXT: or.b64 %rd15, %rd14, %rd13;
+; SM90-NEXT: setp.ne.b64 %p1, %rd15, 0;
+; SM90-NEXT: mov.b64 %rd22, %rd5;
+; SM90-NEXT: mov.b64 %rd23, %rd6;
+; SM90-NEXT: @%p1 bra $L__BB278_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd16, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd16};
+; SM90-NEXT: mov.b128 swap, {%rd16, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd24, %rd25}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB278_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd17, %rd24, %rd3;
+; SM90-NEXT: max.f64 %rd18, %rd25, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd25};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd19, %rd8, %rd25;
+; SM90-NEXT: xor.b64 %rd20, %rd7, %rd24;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p2, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd24, %rd7;
+; SM90-NEXT: mov.b64 %rd25, %rd8;
+; SM90-NEXT: @%p2 bra $L__BB278_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<50>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB279_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd43;
+; SM90-NEXT: mov.b64 %rd9, %rd42;
+; SM90-NEXT: max.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: max.f64 %rd20, %rd10, %rd6;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd19, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd42, %rd43}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd43, %rd10;
+; SM90-NEXT: xor.b64 %rd22, %rd42, %rd9;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd23, 0;
+; SM90-NEXT: @%p1 bra $L__BB279_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB279_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd45;
+; SM90-NEXT: mov.b64 %rd11, %rd44;
+; SM90-NEXT: max.f64 %rd25, %rd11, %rd7;
+; SM90-NEXT: max.f64 %rd26, %rd12, %rd8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd25, %rd26};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd44, %rd45}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd27, %rd45, %rd12;
+; SM90-NEXT: xor.b64 %rd28, %rd44, %rd11;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p2, %rd29, 0;
+; SM90-NEXT: @%p2 bra $L__BB279_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd30, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd46, %rd47}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB279_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd31, %rd46, %rd1;
+; SM90-NEXT: max.f64 %rd32, %rd47, %rd2;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd47};
+; SM90-NEXT: mov.b128 swap, {%rd31, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd33, %rd14, %rd47;
+; SM90-NEXT: xor.b64 %rd34, %rd13, %rd46;
+; SM90-NEXT: or.b64 %rd35, %rd34, %rd33;
+; SM90-NEXT: setp.ne.b64 %p3, %rd35, 0;
+; SM90-NEXT: mov.b64 %rd46, %rd13;
+; SM90-NEXT: mov.b64 %rd47, %rd14;
+; SM90-NEXT: @%p3 bra $L__BB279_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd36, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd36};
+; SM90-NEXT: mov.b128 swap, {%rd36, %rd36};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd48, %rd49}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: $L__BB279_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f64 %rd37, %rd48, %rd3;
+; SM90-NEXT: max.f64 %rd38, %rd49, %rd4;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd48, %rd49};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd38};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd39, %rd16, %rd49;
+; SM90-NEXT: xor.b64 %rd40, %rd15, %rd48;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p4, %rd41, 0;
+; SM90-NEXT: mov.b64 %rd48, %rd15;
+; SM90-NEXT: mov.b64 %rd49, %rd16;
+; SM90-NEXT: @%p4 bra $L__BB279_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM90-NEXT: min.f64 %rd4, %rd9, %rd1;
+; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM90-NEXT: mov.b64 %rd9, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB280_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd4, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd5, -9223372036854775808;
+; SM90-NEXT: $L__BB281_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
+; SM90-NEXT: min.f64 %rd7, %rd20, %rd4;
+; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd20, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
+; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
+; SM90-NEXT: min.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd21, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
+; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
+; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd20, %rd1;
+; SM90-NEXT: mov.b64 %rd21, %rd2;
+; SM90-NEXT: @%p9 bra $L__BB281_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<19>;
+; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd2, -9223372036854775808;
+; SM90-NEXT: $L__BB282_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
+; SM90-NEXT: min.f64 %rd11, %rd38, %rd1;
+; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd38, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
+; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
+; SM90-NEXT: min.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd39, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
+; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
+; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
+; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
+; SM90-NEXT: mov.b64 %rd38, %rd5;
+; SM90-NEXT: mov.b64 %rd39, %rd6;
+; SM90-NEXT: @%p9 bra $L__BB282_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd3, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p16, %rd4, -9223372036854775808;
+; SM90-NEXT: $L__BB282_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
+; SM90-NEXT: min.f64 %rd25, %rd40, %rd3;
+; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd40, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
+; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
+; SM90-NEXT: min.f64 %rd30, %rd41, %rd4;
+; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd41, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
+; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
+; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
+; SM90-NEXT: mov.b64 %rd40, %rd7;
+; SM90-NEXT: mov.b64 %rd41, %rd8;
+; SM90-NEXT: @%p18 bra $L__BB282_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<37>;
+; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
+; SM90-NEXT: $L__BB283_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd75;
+; SM90-NEXT: mov.b64 %rd9, %rd74;
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
+; SM90-NEXT: min.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
+; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
+; SM90-NEXT: min.f64 %rd24, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd10, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
+; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
+; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
+; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
+; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
+; SM90-NEXT: @%p9 bra $L__BB283_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd7, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p16, %rd8, -9223372036854775808;
+; SM90-NEXT: $L__BB283_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd77;
+; SM90-NEXT: mov.b64 %rd11, %rd76;
+; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: min.f64 %rd33, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd11, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
+; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
+; SM90-NEXT: min.f64 %rd38, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd12, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
+; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
+; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
+; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
+; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
+; SM90-NEXT: @%p18 bra $L__BB283_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p21, %rd1, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p25, %rd2, -9223372036854775808;
+; SM90-NEXT: $L__BB283_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
+; SM90-NEXT: min.f64 %rd47, %rd78, %rd1;
+; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
+; SM90-NEXT: setp.eq.b64 %p20, %rd78, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
+; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
+; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
+; SM90-NEXT: min.f64 %rd52, %rd79, %rd2;
+; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
+; SM90-NEXT: setp.eq.b64 %p24, %rd79, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
+; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
+; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
+; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
+; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
+; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
+; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
+; SM90-NEXT: mov.b64 %rd78, %rd13;
+; SM90-NEXT: mov.b64 %rd79, %rd14;
+; SM90-NEXT: @%p27 bra $L__BB283_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p30, %rd3, -9223372036854775808;
+; SM90-NEXT: setp.eq.b64 %p34, %rd4, -9223372036854775808;
+; SM90-NEXT: $L__BB283_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
+; SM90-NEXT: min.f64 %rd61, %rd80, %rd3;
+; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
+; SM90-NEXT: setp.eq.b64 %p29, %rd80, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
+; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
+; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
+; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
+; SM90-NEXT: min.f64 %rd66, %rd81, %rd4;
+; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
+; SM90-NEXT: setp.eq.b64 %p33, %rd81, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
+; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
+; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
+; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
+; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
+; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
+; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
+; SM90-NEXT: mov.b64 %rd80, %rd15;
+; SM90-NEXT: mov.b64 %rd81, %rd16;
+; SM90-NEXT: @%p36 bra $L__BB283_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<6>;
+; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
+; SM90-NEXT: max.f64 %rd4, %rd9, %rd1;
+; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
+; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
+; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
+; SM90-NEXT: mov.b64 %rd9, %rd2;
+; SM90-NEXT: @%p5 bra $L__BB284_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
+ ret <1 x double> %retval
+}
+
+define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<10>;
+; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd6, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
+; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd4, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd5, 0;
+; SM90-NEXT: $L__BB285_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
+; SM90-NEXT: max.f64 %rd7, %rd20, %rd4;
+; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd20, 0;
+; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
+; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
+; SM90-NEXT: max.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd21, 0;
+; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
+; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
+; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
+; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
+; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
+; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
+; SM90-NEXT: mov.b64 %rd20, %rd1;
+; SM90-NEXT: mov.b64 %rd21, %rd2;
+; SM90-NEXT: @%p9 bra $L__BB285_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
+ ret <2 x double> %retval
+}
+
+define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<19>;
+; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd10, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd2, 0;
+; SM90-NEXT: $L__BB286_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
+; SM90-NEXT: max.f64 %rd11, %rd38, %rd1;
+; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd38, 0;
+; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
+; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
+; SM90-NEXT: max.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd39, 0;
+; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
+; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
+; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
+; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
+; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
+; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
+; SM90-NEXT: mov.b64 %rd38, %rd5;
+; SM90-NEXT: mov.b64 %rd39, %rd6;
+; SM90-NEXT: @%p9 bra $L__BB286_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd3, 0;
+; SM90-NEXT: setp.eq.b64 %p16, %rd4, 0;
+; SM90-NEXT: $L__BB286_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
+; SM90-NEXT: max.f64 %rd25, %rd40, %rd3;
+; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd40, 0;
+; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
+; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
+; SM90-NEXT: max.f64 %rd30, %rd41, %rd4;
+; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd41, 0;
+; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
+; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
+; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
+; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
+; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
+; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
+; SM90-NEXT: mov.b64 %rd40, %rd7;
+; SM90-NEXT: mov.b64 %rd41, %rd8;
+; SM90-NEXT: @%p18 bra $L__BB286_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
+ ret <4 x double> %retval
+}
+
+define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f64_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<37>;
+; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd18, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
+; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM90-NEXT: setp.eq.b64 %p7, %rd6, 0;
+; SM90-NEXT: $L__BB287_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd10, %rd75;
+; SM90-NEXT: mov.b64 %rd9, %rd74;
+; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
+; SM90-NEXT: max.f64 %rd19, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
+; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
+; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
+; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
+; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
+; SM90-NEXT: max.f64 %rd24, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
+; SM90-NEXT: setp.eq.b64 %p6, %rd10, 0;
+; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
+; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
+; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
+; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
+; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
+; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
+; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
+; SM90-NEXT: @%p9 bra $L__BB287_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p12, %rd7, 0;
+; SM90-NEXT: setp.eq.b64 %p16, %rd8, 0;
+; SM90-NEXT: $L__BB287_3: // %atomicrmw.start6
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: mov.b64 %rd12, %rd77;
+; SM90-NEXT: mov.b64 %rd11, %rd76;
+; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: max.f64 %rd33, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd11, 0;
+; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
+; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
+; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
+; SM90-NEXT: max.f64 %rd38, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
+; SM90-NEXT: setp.eq.b64 %p15, %rd12, 0;
+; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
+; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
+; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
+; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
+; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
+; SM90-NEXT: @%p18 bra $L__BB287_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end5
+; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p21, %rd1, 0;
+; SM90-NEXT: setp.eq.b64 %p25, %rd2, 0;
+; SM90-NEXT: $L__BB287_5: // %atomicrmw.start15
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
+; SM90-NEXT: max.f64 %rd47, %rd78, %rd1;
+; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
+; SM90-NEXT: setp.eq.b64 %p20, %rd78, 0;
+; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
+; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
+; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
+; SM90-NEXT: max.f64 %rd52, %rd79, %rd2;
+; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
+; SM90-NEXT: setp.eq.b64 %p24, %rd79, 0;
+; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
+; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
+; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
+; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
+; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
+; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
+; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
+; SM90-NEXT: mov.b64 %rd78, %rd13;
+; SM90-NEXT: mov.b64 %rd79, %rd14;
+; SM90-NEXT: @%p27 bra $L__BB287_5;
+; SM90-NEXT: // %bb.6: // %atomicrmw.end14
+; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: setp.eq.b64 %p30, %rd3, 0;
+; SM90-NEXT: setp.eq.b64 %p34, %rd4, 0;
+; SM90-NEXT: $L__BB287_7: // %atomicrmw.start20
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
+; SM90-NEXT: max.f64 %rd61, %rd80, %rd3;
+; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
+; SM90-NEXT: setp.eq.b64 %p29, %rd80, 0;
+; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
+; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
+; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
+; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
+; SM90-NEXT: max.f64 %rd66, %rd81, %rd4;
+; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
+; SM90-NEXT: setp.eq.b64 %p33, %rd81, 0;
+; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
+; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
+; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
+; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
+; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
+; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
+; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
+; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
+; SM90-NEXT: mov.b64 %rd80, %rd15;
+; SM90-NEXT: mov.b64 %rd81, %rd16;
+; SM90-NEXT: @%p36 bra $L__BB287_7;
+; SM90-NEXT: // %bb.8: // %atomicrmw.end19
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
+ ret <8 x double> %retval
+}
+
+define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB292_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB292_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB293_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB293_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB294_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB294_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fsub_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB295_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB295_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB296_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB296_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB297_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB297_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB298_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB298_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmin_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB299_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB299_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB300_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB300_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB301_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB301_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB302_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB302_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmax_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB303_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB303_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB304_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB304_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB305_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB305_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB306_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB306_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB307_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB307_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB308_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB308_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
+ ret <1 x half> %retval
+}
+
+define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB309_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB309_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
+ ret <2 x half> %retval
+}
+
+define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB310_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.f16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB310_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
+ ret <4 x half> %retval
+}
+
+define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8f16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB311_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.f16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.f16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.f16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.f16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB311_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
+ ret <8 x half> %retval
+}
+
+define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs5, [%rd1], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs2;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fadd_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b16 %rs<17>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
+; SM90-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs9, [%rd1], %rs7;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs10, [%rd1+2], %rs8;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs11, [%rd1+4], %rs5;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs12, [%rd1+6], %rs6;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs13, [%rd1+8], %rs3;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs14, [%rd1+10], %rs4;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs15, [%rd1+12], %rs1;
+; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs16, [%rd1+14], %rs2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: mov.b32 %r5, {%rs9, %rs10};
+; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
+; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
+; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB316_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: sub.rn.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB316_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB317_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB317_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB318_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: sub.rn.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB318_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fsub_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB319_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: sub.rn.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: sub.rn.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: sub.rn.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: sub.rn.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB319_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB320_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB320_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB321_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB321_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB322_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB322_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmin_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB323_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB323_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB324_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB324_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB325_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB325_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB326_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB326_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmax_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB327_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB327_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB328_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: min.NaN.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB328_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB329_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB329_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB330_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: min.NaN.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB330_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fminimum_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB331_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: min.NaN.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: min.NaN.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: min.NaN.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: min.NaN.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB331_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<4>;
+; SM90-NEXT: .reg .b32 %r<15>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NEXT: mov.b32 %r6, 65535;
+; SM90-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NEXT: not.b32 %r2, %r7;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NEXT: $L__BB332_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NEXT: max.NaN.bf16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NEXT: mov.b32 %r14, %r3;
+; SM90-NEXT: @%p1 bra $L__BB332_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
+ ret <1 x bfloat> %retval
+}
+
+define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NEXT: $L__BB333_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r3, %r4, %r2;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NEXT: mov.b32 %r4, %r1;
+; SM90-NEXT: @%p1 bra $L__BB333_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
+ ret <2 x bfloat> %retval
+}
+
+define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB334_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r3, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r3;
+; SM90-NEXT: max.NaN.bf16x2 %r4, %r6, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r4;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB334_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
+ ret <4 x bfloat> %retval
+}
+
+define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
+; SM90-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<26>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB335_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: max.NaN.bf16x2 %r5, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd5, %r5;
+; SM90-NEXT: max.NaN.bf16x2 %r6, %r10, %r2;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r6;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: max.NaN.bf16x2 %r7, %r11, %r3;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r7;
+; SM90-NEXT: max.NaN.bf16x2 %r8, %r12, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r8;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r10;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: cvt.u64.u32 %rd17, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r12;
+; SM90-NEXT: shl.b64 %rd19, %rd18, 32;
+; SM90-NEXT: or.b64 %rd20, %rd17, %rd19;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd16, %rd20};
+; SM90-NEXT: mov.b128 swap, {%rd8, %rd12};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd21, %rd22}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd23, %rd22, %rd20;
+; SM90-NEXT: xor.b64 %rd24, %rd21, %rd16;
+; SM90-NEXT: or.b64 %rd25, %rd24, %rd23;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd22;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd22;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd21;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd21;
+; SM90-NEXT: setp.ne.b64 %p1, %rd25, 0;
+; SM90-NEXT: @%p1 bra $L__BB335_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
+ ret <8 x bfloat> %retval
+}
+
+define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_monotonic_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB336_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB336_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_monotonic_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB337_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB337_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_monotonic_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB338_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB338_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_monotonic_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB339_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB339_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_acquire_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB340_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB340_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_acquire_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB341_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB341_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_acquire_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB342_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB342_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_acquire_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB343_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB343_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_release_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB344_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB344_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_release_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB345_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB345_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_release_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB346_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB346_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_release_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB347_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB347_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<16>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
+; SM90-NEXT: $L__BB348_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: add.s32 %r10, %r15, %r4;
+; SM90-NEXT: and.b32 %r11, %r10, %r2;
+; SM90-NEXT: and.b32 %r12, %r15, %r3;
+; SM90-NEXT: or.b32 %r13, %r12, %r11;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
+; SM90-NEXT: mov.b32 %r15, %r5;
+; SM90-NEXT: @%p1 bra $L__BB348_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r14, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<10>;
+; SM90-NEXT: .reg .b32 %r<18>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM90-NEXT: $L__BB349_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r17, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: and.b16 %rs8, %rs5, 255;
+; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: shl.b32 %r13, %r12, %r1;
+; SM90-NEXT: and.b32 %r14, %r17, %r2;
+; SM90-NEXT: or.b32 %r15, %r14, %r13;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM90-NEXT: mov.b32 %r17, %r3;
+; SM90-NEXT: @%p1 bra $L__BB349_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r16, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<13>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: $L__BB350_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
+; SM90-NEXT: mov.b32 %r18, %r1;
+; SM90-NEXT: @%p1 bra $L__BB350_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: add_seq_cst_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<25>;
+; SM90-NEXT: .reg .b32 %r<35>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
+; SM90-NEXT: $L__BB351_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
+; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
+; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
+; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
+; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
+; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
+; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
+; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
+; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
+; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
+; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
+; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
+; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
+; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
+; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
+; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
+; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
+; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
+; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
+; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
+; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
+; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
+; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
+; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
+; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
+; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
+; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
+; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB351_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_monotonic_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_monotonic_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_monotonic_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_monotonic_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_acquire_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_acquire_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_acquire_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_acquire_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_release_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_release_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<3>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<5>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: add_seq_cst_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
+
+define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB368_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB369_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB370_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_monotonic_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB371_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_acquire_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB372_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_acquire_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB373_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_acquire_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB374_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_acquire_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB375_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_release_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB376_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_release_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB377_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB377_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_release_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB378_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB378_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_release_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB379_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB379_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
+ ret <8 x i8> %retval
+}
+
+define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v1i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<17>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: and.b32 %r8, %r7, 3;
+; SM90-NEXT: shl.b32 %r1, %r8, 3;
+; SM90-NEXT: mov.b32 %r9, 255;
+; SM90-NEXT: shl.b32 %r2, %r9, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: shl.b32 %r4, %r6, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
+; SM90-NEXT: @%p1 bra $L__BB380_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
+ ret <1 x i8> %retval
+}
+
+define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v2i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b16 %rs<9>;
+; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
+; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: and.b32 %r6, %r5, 3;
+; SM90-NEXT: shl.b32 %r1, %r6, 3;
+; SM90-NEXT: mov.b32 %r7, 65535;
+; SM90-NEXT: shl.b32 %r8, %r7, %r1;
+; SM90-NEXT: not.b32 %r2, %r8;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: $L__BB381_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: shr.u32 %r9, %r18, %r1;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
+; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
+; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
+; SM90-NEXT: and.b32 %r11, %r10, %r4;
+; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
+; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
+; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
+; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
+; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
+; SM90-NEXT: shl.b32 %r14, %r13, %r1;
+; SM90-NEXT: and.b32 %r15, %r18, %r2;
+; SM90-NEXT: or.b32 %r16, %r15, %r14;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
+; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
+; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: @%p1 bra $L__BB381_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ ret <2 x i8> %retval
+}
+
+define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v4i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB382_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r2;
+; SM90-NEXT: xor.b32 %r4, %r3, -1;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
+; SM90-NEXT: mov.b32 %r5, %r1;
+; SM90-NEXT: @%p1 bra $L__BB382_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ ret <4 x i8> %retval
+}
+
+define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM90-LABEL: nand_seq_cst_v8i8_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<9>;
+; SM90-NEXT: .reg .b64 %rd<12>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
+; SM90-NEXT: $L__BB383_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r8, %r2;
+; SM90-NEXT: and.b32 %r4, %r7, %r1;
+; SM90-NEXT: xor.b32 %r5, %r4, -1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
+; SM90-NEXT: xor.b32 %r6, %r3, -1;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
+; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
+; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
+; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
+; SM90-NEXT: @%p1 bra $L__BB383_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ ret <8 x i8> %retval
+}
+
+define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB384_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB385_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB386_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_monotonic_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB387_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB387_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB387_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_acquire_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB388_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_acquire_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB389_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_acquire_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB390_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_acquire_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB391_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB391_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB391_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_release_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB392_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_release_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB393_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_release_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB394_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_release_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.release.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB395_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB395_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB395_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
+ ret <8 x i32> %retval
+}
+
+define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v1i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<6>;
+; SM90-NEXT: .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
+; SM90-NEXT: $L__BB396_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: and.b32 %r3, %r5, %r1;
+; SM90-NEXT: not.b32 %r4, %r3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
+; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
+; SM90-NEXT: mov.b32 %r5, %r2;
+; SM90-NEXT: @%p1 bra $L__BB396_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
+ ret <1 x i32> %retval
+}
+
+define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v2i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<7>;
+; SM90-NEXT: .reg .b64 %rd<13>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
+; SM90-NEXT: $L__BB397_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
+; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
+; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: and.b32 %r3, %r6, %r2;
+; SM90-NEXT: and.b32 %r4, %r5, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
+; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
+; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
+; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
+; SM90-NEXT: not.b64 %rd11, %rd10;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
+; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
+; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
+; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
+; SM90-NEXT: @%p1 bra $L__BB397_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
+ ret <2 x i32> %retval
+}
+
+define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v4i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<2>;
+; SM90-NEXT: .reg .b32 %r<13>;
+; SM90-NEXT: .reg .b64 %rd<28>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
+; SM90-NEXT: $L__BB398_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r5, %r12, %r4;
+; SM90-NEXT: and.b32 %r6, %r11, %r3;
+; SM90-NEXT: and.b32 %r7, %r10, %r2;
+; SM90-NEXT: and.b32 %r8, %r9, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
+; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB398_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
+ ret <4 x i32> %retval
+}
+
+define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
+; SM90-LABEL: nand_seq_cst_v8i32_global_cta(
+; SM90: {
+; SM90-NEXT: .reg .pred %p<3>;
+; SM90-NEXT: .reg .b32 %r<25>;
+; SM90-NEXT: .reg .b64 %rd<54>;
+; SM90-EMPTY:
+; SM90-NEXT: // %bb.0:
+; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
+; SM90-NEXT: fence.sc.cta;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1+16];
+; SM90-NEXT: mov.b64 %rd2, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
+; SM90-NEXT: $L__BB399_1: // %atomicrmw.start
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
+; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
+; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
+; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
+; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
+; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
+; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
+; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
+; SM90-NEXT: and.b32 %r9, %r20, %r4;
+; SM90-NEXT: and.b32 %r10, %r19, %r3;
+; SM90-NEXT: and.b32 %r11, %r18, %r2;
+; SM90-NEXT: and.b32 %r12, %r17, %r1;
+; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
+; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
+; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
+; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
+; SM90-NEXT: not.b64 %rd17, %rd16;
+; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
+; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
+; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
+; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
+; SM90-NEXT: not.b64 %rd22, %rd21;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
+; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
+; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
+; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
+; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
+; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
+; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
+; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
+; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
+; SM90-NEXT: @%p1 bra $L__BB399_1;
+; SM90-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NEXT: mov.b64 %rd28, 0;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
+; SM90-NEXT: $L__BB399_3: // %atomicrmw.start2
+; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
+; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
+; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
+; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
+; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
+; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
+; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
+; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
+; SM90-NEXT: and.b32 %r13, %r24, %r8;
+; SM90-NEXT: and.b32 %r14, %r23, %r7;
+; SM90-NEXT: and.b32 %r15, %r22, %r6;
+; SM90-NEXT: and.b32 %r16, %r21, %r5;
+; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
+; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
+; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
+; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
+; SM90-NEXT: not.b64 %rd43, %rd42;
+; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
+; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
+; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
+; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
+; SM90-NEXT: not.b64 %rd48, %rd47;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
+; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
+; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
+; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
+; SM90-NEXT: }
+; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
+; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
+; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
+; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
+; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
+; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
+; SM90-NEXT: @%p2 bra $L__BB399_3;
+; SM90-NEXT: // %bb.4: // %atomicrmw.end1
+; SM90-NEXT: fence.acquire.cta;
+; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: ret;
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
+ ret <8 x i32> %retval
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py
index 966d62bf1b03a..ff007030e785f 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw.py
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py
@@ -9,6 +9,8 @@
ORDERINGS = ["monotonic", "acquire", "release", "acq_rel", "seq_cst"]
+VECTOR_SIZES = [1, 2, 4, 8]
+
INTEGER_OPERATIONS = [
"xchg",
"add",
@@ -29,6 +31,13 @@
FLOATING_POINT_OPERATIONS = ["fadd", "fsub", "fmin", "fmax", "fminimum", "fmaximum"]
+DATATYPE_SUFFIXES = {
+ "float": "f32",
+ "double": "f64",
+ "half": "f16",
+ "bfloat": "bf16",
+}
+
ADDRSPACE_NUM_TO_ADDRSPACE = {0: "generic", 1: "global", 3: "shared"}
# A small spread of operations used to exercise the scope and address-space
@@ -48,7 +57,15 @@
atomicrmw_func = Template(
"""define ${datatype} @${operation}_${ordering}_${datatype}_${addrspace}_${ptx_scope}(ptr${addrspace_cast} %addr, ${datatype} %val) {
- %retval = atomicrmw ${operation} ptr ${addrspace_cast} %addr, ${datatype} %val syncscope(\"${llvm_scope}\") ${ordering}
+ %retval = atomicrmw ${operation} ptr ${addrspace_cast} %addr, ${datatype} %val syncscope(\"${llvm_scope}\") ${ordering}
+ ret $datatype %retval
+}
+"""
+)
+
+atomicrmw_elementwise_func = Template(
+ """define ${datatype} @${operation}_${ordering}_${datatype_name}_${addrspace}_${ptx_scope}(ptr${addrspace_cast} %addr, ${datatype} %val) {
+ %retval = atomicrmw elementwise ${operation} ptr ${addrspace_cast} %addr, ${datatype} %val syncscope(\"${llvm_scope}\") ${ordering}
ret $datatype %retval
}
"""
@@ -59,6 +76,7 @@
# the SM${sm} prefix; only the FTZ-sensitive ops diverge into SM${sm}-NOFTZ /
# SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in
# atomicrmw-allow-ftz-atomics.ll.)
+
run_statement = Template(
"""; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ
; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ
@@ -75,100 +93,133 @@ def get_addrspace_cast(addrspace):
return " addrspace({})".format(str(addrspace))
+def get_vector_datatype(datatype, size):
+ return "<{} x {}>".format(size, datatype)
+
+
+def get_vector_datatype_name(datatype, size):
+ return "v{}{}".format(size, DATATYPE_SUFFIXES.get(datatype, datatype))
+
+
+def print_atomicrmw(
+ fp, operation, datatype, ordering, addrspace, llvm_scope, vector_size=None
+):
+ if vector_size is None:
+ template = atomicrmw_func
+ ir_datatype = datatype
+ datatype_name = datatype
+ else:
+ template = atomicrmw_elementwise_func
+ ir_datatype = get_vector_datatype(datatype, vector_size)
+ datatype_name = get_vector_datatype_name(datatype, vector_size)
+
+ print(
+ template.substitute(
+ operation=operation,
+ ordering=ordering,
+ datatype=ir_datatype,
+ datatype_name=datatype_name,
+ addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
+ ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
+ llvm_scope=llvm_scope,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ ),
+ file=fp,
+ )
+
+
if __name__ == "__main__":
for sm, ptx in TEST_SM_ARCH_PAIRS:
- # Slice 1: Keep addrspace, llvm_scope, ordering fixed, generate all possible operations and sizes
- with open("atomicrmw-sm{}.ll".format(str(sm)), "w") as fp:
- print(run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0), file=fp)
- # Integer operations
- addrspace, llvm_scope, ordering = 1, "block", "acq_rel"
- for operation, datatype in product(
- INTEGER_OPERATIONS, ["i8", "i16", "i32", "i64"]
- ):
+ for elementwise in [False, True]:
+ vector_sizes = VECTOR_SIZES if elementwise else [None]
+ filename_suffix = "-elementwise" if elementwise else ""
+ with open("atomicrmw{}-sm{}.ll".format(filename_suffix, sm), "w") as fp:
print(
- atomicrmw_func.substitute(
- operation=operation,
- ordering=ordering,
- datatype=datatype,
- addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
- ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
- llvm_scope=llvm_scope,
- addrspace_cast=get_addrspace_cast(addrspace),
- ),
+ run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0),
file=fp,
)
- # Floating point add
- for datatype, operation in product(
- ["float", "double", "half", "bfloat"], FLOATING_POINT_OPERATIONS
- ):
- print(
- atomicrmw_func.substitute(
- operation=operation,
- ordering=ordering,
- datatype=datatype,
- addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
- ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
- llvm_scope=llvm_scope,
- addrspace_cast=get_addrspace_cast(addrspace),
- ),
- file=fp,
- )
-
- # Slice 2: Keep addrspace, llvm_scope fixed, and generate all possible orderings for operations add and nand.
- # add is natively supported for larger bitwidths, while nand is emulated always
- addrspace, llvm_scope = 1, "block"
- for operation, datatype, ordering in product(
- ["add", "nand"], ["i8", "i32"], ORDERINGS
- ):
- if addrspace == 1 and llvm_scope == "block" and ordering == "acq_rel":
- # These are a part of Slice 1
- continue
- print(
- atomicrmw_func.substitute(
- operation=operation,
- ordering=ordering,
- datatype=datatype,
- addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
- addrspace_cast=get_addrspace_cast(addrspace),
- ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
- llvm_scope=llvm_scope,
- ),
- file=fp,
- )
+ # Slice 1: Keep addrspace, llvm_scope, and ordering fixed while
+ # generating every operation and type combination.
+ addrspace, llvm_scope, ordering = 1, "block", "acq_rel"
+ for operation, datatype, vector_size in product(
+ INTEGER_OPERATIONS,
+ ["i8", "i16", "i32", "i64"],
+ vector_sizes,
+ ):
+ print_atomicrmw(
+ fp,
+ operation,
+ datatype,
+ ordering,
+ addrspace,
+ llvm_scope,
+ vector_size,
+ )
- # Slice 3: Keep addrspace (global) and ordering fixed, vary the scope
- # qualifier. block/global is already covered by Slice 1.
- addrspace, ordering = 1, "acq_rel"
- for llvm_scope in [s for s in SCOPE_LLVM_TO_PTX if s != "block"]:
- for operation, datatype in REPRESENTATIVE_OPS:
- print(
- atomicrmw_func.substitute(
- operation=operation,
- ordering=ordering,
- datatype=datatype,
- addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
- addrspace_cast=get_addrspace_cast(addrspace),
- ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
- llvm_scope=llvm_scope,
- ),
- file=fp,
+ # Floating-point operations.
+ for datatype, operation, vector_size in product(
+ ["float", "double", "half", "bfloat"],
+ FLOATING_POINT_OPERATIONS,
+ vector_sizes,
+ ):
+ print_atomicrmw(
+ fp,
+ operation,
+ datatype,
+ ordering,
+ addrspace,
+ llvm_scope,
+ vector_size,
)
- # Slice 4: Keep scope (block) and ordering fixed, vary the address-space
- # qualifier. global is already covered by Slice 1.
- llvm_scope, ordering = "block", "acq_rel"
- for addrspace in [a for a in ADDRSPACE_NUM_TO_ADDRSPACE if a != 1]:
- for operation, datatype in REPRESENTATIVE_OPS:
- print(
- atomicrmw_func.substitute(
- operation=operation,
- ordering=ordering,
- datatype=datatype,
- addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace],
- addrspace_cast=get_addrspace_cast(addrspace),
- ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope],
- llvm_scope=llvm_scope,
- ),
- file=fp,
+ # Slice 2: Keep addrspace and llvm_scope fixed, and generate all
+ # orderings for add and nand. add is natively supported for larger
+ # bitwidths, while nand is always emulated.
+ addrspace, llvm_scope = 1, "block"
+ for operation, datatype, ordering, vector_size in product(
+ ["add", "nand"], ["i8", "i32"], ORDERINGS, vector_sizes
+ ):
+ if ordering == "acq_rel":
+ # These cases are part of Slice 1.
+ continue
+ print_atomicrmw(
+ fp,
+ operation,
+ datatype,
+ ordering,
+ addrspace,
+ llvm_scope,
+ vector_size,
)
+
+ if elementwise:
+ continue
+
+ # Slice 3: Keep addrspace (global) and ordering fixed, vary the
+ # scope qualifier. block/global is already covered by Slice 1.
+ addrspace, ordering = 1, "acq_rel"
+ for llvm_scope in [s for s in SCOPE_LLVM_TO_PTX if s != "block"]:
+ for operation, datatype in REPRESENTATIVE_OPS:
+ print_atomicrmw(
+ fp,
+ operation,
+ datatype,
+ ordering,
+ addrspace,
+ llvm_scope,
+ )
+
+ # Slice 4: Keep scope (block) and ordering fixed, vary the
+ # address-space qualifier. global is already covered by Slice 1.
+ llvm_scope, ordering = "block", "acq_rel"
+ for addrspace in [a for a in ADDRSPACE_NUM_TO_ADDRSPACE if a != 1]:
+ for operation, datatype in REPRESENTATIVE_OPS:
+ print_atomicrmw(
+ fp,
+ operation,
+ datatype,
+ ordering,
+ addrspace,
+ llvm_scope,
+ )
>From 36a7bf9c628b0ab265cd3fef7f5b7f57386f5bb4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 18 May 2026 20:17:05 +0000
Subject: [PATCH 03/15] isStrongerThanMonotonic
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 1c6307dc8856f..030eb0dd6d5a3 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -389,13 +389,11 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
if (ExpansionKind ==
TargetLoweringBase::AtomicExpansionKind::Elementwise) {
// If the target wants fence-based lowering for this elementwise RMW,
- // insert the fences before splitting and downgrade the RMW ordering
- // first. The split operations inherit the downgraded ordering, so they
+ // insert the fences and downgrade the RMW ordering before splitting.
+ // The split operations inherit the downgraded ordering, so they
// do not each get their own fence pair.
tryInsertFencesForAtomic(
- RMWI,
- isReleaseOrStronger(RMWI->getOrdering()) ||
- isAcquireOrStronger(RMWI->getOrdering()),
+ RMWI, isStrongerThanMonotonic(RMWI->getOrdering()),
TLI->atomicOperationOrderAfterFenceSplit(RMWI));
expandElementwiseAtomicRMW(RMWI);
return true;
@@ -415,9 +413,7 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
MadeChange |= tryInsertFencesForAtomic(
- RMWI,
- isReleaseOrStronger(RMWI->getOrdering()) ||
- isAcquireOrStronger(RMWI->getOrdering()),
+ RMWI, isStrongerThanMonotonic(RMWI->getOrdering()),
TLI->atomicOperationOrderAfterFenceSplit(RMWI));
// There are two different ways of expanding RMW instructions:
>From b45ebb8dd4b145692567a95dad427318054e4b4e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 18 May 2026 23:18:36 +0000
Subject: [PATCH 04/15] antonio feedback
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 030eb0dd6d5a3..28eeaa1503a83 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -20,6 +20,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -709,19 +710,17 @@ void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
// N > 2: split into two <N/2 x T> elementwise atomicrmws and recurse via
// processAtomicInstr().
- assert(NumLanes % 2 == 0 &&
+ assert(isPowerOf2_32(NumLanes) &&
"elementwise atomicrmw vector length must be a power of two");
const unsigned HalfLanes = NumLanes / 2;
auto *HalfVecTy = FixedVectorType::get(LaneTy, HalfLanes);
const uint64_t HalfBytes = DL->getTypeStoreSize(HalfVecTy).getFixedValue();
- SmallVector<int, 16> LoMask(HalfLanes), HiMask(HalfLanes);
- for (unsigned I = 0; I < HalfLanes; ++I) {
- LoMask[I] = static_cast<int>(I);
- HiMask[I] = static_cast<int>(HalfLanes + I);
- }
- Value *LoVal = Builder.CreateShuffleVector(Val, LoMask, "lo.val");
- Value *HiVal = Builder.CreateShuffleVector(Val, HiMask, "hi.val");
+ Value *LoVal =
+ Builder.CreateShuffleVector(Val, createSequentialMask(0, HalfLanes, 0),
+ "lo.val");
+ Value *HiVal = Builder.CreateShuffleVector(
+ Val, createSequentialMask(HalfLanes, HalfLanes, 0), "hi.val");
Value *HiPtr = Builder.CreateInBoundsGEP(HalfVecTy, Ptr, IdxOne, "hi.ptr");
Align LoAlign = AI->getAlign();
@@ -732,10 +731,8 @@ void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
AtomicRMWInst *HiRMW =
CreateRMWInstruction(HiPtr, HiVal, HiAlign, /*Elementwise=*/true);
- SmallVector<int, 16> Mask(NumLanes);
- for (unsigned I = 0; I < NumLanes; ++I)
- Mask[I] = static_cast<int>(I);
- Value *Result = Builder.CreateShuffleVector(LoRMW, HiRMW, Mask, "old");
+ Value *Result = concatenateVectors(Builder, {LoRMW, HiRMW});
+ Result->setName("old");
AI->replaceAllUsesWith(Result);
AI->eraseFromParent();
>From 612b7570243934a26242af42babf93c035b719fb Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 18 May 2026 23:19:14 +0000
Subject: [PATCH 05/15] format
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 28eeaa1503a83..af09c71959ed9 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -716,9 +716,8 @@ void AtomicExpandImpl::expandElementwiseAtomicRMW(AtomicRMWInst *AI) {
auto *HalfVecTy = FixedVectorType::get(LaneTy, HalfLanes);
const uint64_t HalfBytes = DL->getTypeStoreSize(HalfVecTy).getFixedValue();
- Value *LoVal =
- Builder.CreateShuffleVector(Val, createSequentialMask(0, HalfLanes, 0),
- "lo.val");
+ Value *LoVal = Builder.CreateShuffleVector(
+ Val, createSequentialMask(0, HalfLanes, 0), "lo.val");
Value *HiVal = Builder.CreateShuffleVector(
Val, createSequentialMask(HalfLanes, HalfLanes, 0), "hi.val");
>From d05610d8a58864f68213190a4a8a5c4514de76f0 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 29 Jun 2026 21:10:48 +0000
Subject: [PATCH 06/15] Use Expand
---
llvm/include/llvm/CodeGen/TargetLowering.h | 14 ------
llvm/lib/CodeGen/AtomicExpandPass.cpp | 49 +++++++++----------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 21 ++++----
.../X86/expand-atomic-rmw-elementwise.ll | 2 +-
4 files changed, 36 insertions(+), 50 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 057fe9d8da075..6ceb3150e7610 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -270,20 +270,6 @@ class LLVM_ABI TargetLoweringBase {
Expand, // Generic expansion in terms of other atomic operations.
CustomExpand, // Custom target-specific expansion using TLI hooks.
- // Halve an elementwise vector atomicrmw and re-ask the target how to expand
- // each half. Split an `atomicrmw elementwise <N x T>` into either
- // two `atomicrmw elementwise <N/2 x T>` (when N > 2) or two scalar
- // `atomicrmw T` (when N == 2). An `<1 x T>` elementwise RMW is collapsed
- // directly to a scalar `atomicrmw T`. Each resulting atomicrmw is fed back
- // through the expansion pipeline, so the target's
- // `shouldExpandAtomicRMWInIR` is re-queried at the halved width and may
- // return any expansion kind there:
- // - `None` to preserve at that width (e.g. a native vector atomic),
- // - `Elementwise` to keep halving,
- // - `CmpXChg` (or any other kind) to commit to that expansion for the
- // current width.
- Elementwise,
-
// Rewrite to a non-atomic form for use in a known non-preemptible
// environment.
NotAtomic
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index af09c71959ed9..d2bde98dbcbaa 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -385,30 +385,20 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
- if (RMWI->isElementwise()) {
- auto ExpansionKind = TLI->shouldExpandAtomicRMWInIR(RMWI);
- if (ExpansionKind ==
- TargetLoweringBase::AtomicExpansionKind::Elementwise) {
- // If the target wants fence-based lowering for this elementwise RMW,
- // insert the fences and downgrade the RMW ordering before splitting.
- // The split operations inherit the downgraded ordering, so they
- // do not each get their own fence pair.
- tryInsertFencesForAtomic(
- RMWI, isStrongerThanMonotonic(RMWI->getOrdering()),
- TLI->atomicOperationOrderAfterFenceSplit(RMWI));
- expandElementwiseAtomicRMW(RMWI);
- return true;
- }
- }
+ bool IsElementwiseExpand = RMWI->isElementwise() && TLI->shouldExpandAtomicRMWInIR(RMWI) == TargetLoweringBase::AtomicExpansionKind::Expand;
if (!atomicSizeSupported(TLI, RMWI)) {
- expandAtomicRMWToLibcall(RMWI);
- return true;
+ // Elementwise expansion may split an atomicrmw into smaller, supported atomic sizes.
+ if (!IsElementwiseExpand) {
+ expandAtomicRMWToLibcall(RMWI);
+ return true;
+ }
}
bool MadeChange = false;
- if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
- TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
+ if (!IsElementwiseExpand &&
+ TLI->shouldCastAtomicRMWIInIR(RMWI) ==
+ TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
RMWI = convertAtomicXchgToIntegerType(RMWI);
MadeChange = true;
}
@@ -421,8 +411,11 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
// - into a load if it is idempotent
// - into a Cmpxchg/LL-SC loop otherwise
// we try them in that order.
- MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
- tryExpandAtomicRMW(RMWI);
+ if (IsElementwiseExpand)
+ MadeChange |= tryExpandAtomicRMW(RMWI);
+ else
+ MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
+ tryExpandAtomicRMW(RMWI);
return MadeChange;
}
@@ -625,12 +618,12 @@ AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
return NewRMWI;
}
-/// Halve an elementwise vector atomicrmw and feed each half back through the
+/// Generic Expand handling for elementwise vector atomicrmw instructions.
+/// Halve an `atomicrmw elementwise <N x T>` and feed each half back through the
/// normal atomic expansion pipeline. The target's shouldExpandAtomicRMWInIR()
/// is re-queried at the halved width and may return any expansion kind there
-/// (e.g. None to preserve as a native vector atomic, Elementwise to keep
-/// halving, CmpXChg to emit one wide cmpxchg loop at the halved width). Three
-/// cases:
+/// (e.g. None to preserve as a native vector atomic, Expand to keep halving,
+/// CmpXChg to emit one wide cmpxchg loop at the halved width). Three cases:
/// N == 1: collapse directly to one scalar atomicrmw T
/// N == 2: split into two scalar atomicrmw T (low at Ptr, high at
/// gep inbounds T, Ptr, 1); reassemble via two insertelement's.
@@ -980,6 +973,12 @@ bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {
TLI->emitCmpArithAtomicRMWIntrinsic(AI);
return true;
}
+ case TargetLoweringBase::AtomicExpansionKind::Expand:
+ if (AI->isElementwise()) {
+ expandElementwiseAtomicRMW(AI);
+ return true;
+ }
+ llvm_unreachable("Unhandled atomicrmw Expand case");
case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
return lowerAtomicRMWInst(AI);
case TargetLoweringBase::AtomicExpansionKind::CustomExpand:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d1991d076d684..df6fe672c9030 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7475,15 +7475,15 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// Collapse <1 x T> elementwise RMWs to scalar RMWs
if (VecTy->getNumElements() == 1)
- return AtomicExpansionKind::Elementwise;
+ return AtomicExpansionKind::Expand;
- // If the scalar lane op is natively supported, return
- // Elementwise so halving eventually bottoms out at the scalar base
- // case, where this hook returns None for each scalar lane and the
- // scalar `atom.*` instruction is preserved.
+ // If the scalar lane op is natively supported, return Expand so halving
+ // eventually bottoms out at the scalar base case, where this hook returns
+ // None for each scalar lane and the scalar `atom.*` instruction is
+ // preserved.
if (getScalarAtomicRMWExpansion(AI->getOperation(), LaneTy, STI) ==
AtomicExpansionKind::None)
- return AtomicExpansionKind::Elementwise;
+ return AtomicExpansionKind::Expand;
// If the whole vector fits a single native cmpxchg, emit one wide
// cmpxchg loop at the current width.
@@ -7494,7 +7494,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// If the vector is too wide for a single native cmpxchg, halve further to
// emit multiple cmpxchg loops.
- return AtomicExpansionKind::Elementwise;
+ return AtomicExpansionKind::Expand;
}
return getScalarAtomicRMWExpansion(AI->getOperation(),
@@ -7532,7 +7532,7 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
if (auto *RI = dyn_cast<AtomicRMWInst>(I)) {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
return Expansion == AtomicExpansionKind::CmpXChg ||
- Expansion == AtomicExpansionKind::Elementwise ||
+ (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand) ||
RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
}
return false;
@@ -7620,8 +7620,9 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
STI.getMinCmpXchgSizeInBits();
else {
AtomicExpansionKind Expansion = shouldExpandAtomicRMWInIR(RI);
- NeedsTrailingAcquireFence = Expansion == AtomicExpansionKind::CmpXChg ||
- Expansion == AtomicExpansionKind::Elementwise;
+ NeedsTrailingAcquireFence =
+ Expansion == AtomicExpansionKind::CmpXChg ||
+ (RI->isElementwise() && Expansion == AtomicExpansionKind::Expand);
}
if (NeedsTrailingAcquireFence)
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
index a19b5f0e0c1cd..9c5b7857fe1b9 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -2,7 +2,7 @@
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s
; By default, elementwise atomicrmws are treated the same as the non-elementwise versions and are not expanded.
-; Targets in the future can take advantage of elementwise expansion by returning AtomicExpansionKind::Elementwise from shouldExpandAtomicRMWInIR.
+; Targets in the future can take advantage of elementwise expansion by returning AtomicExpansionKind::Expand from shouldExpandAtomicRMWInIR.
; Elementwise atomics work on integer vectors, but normal atomics don't.
define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
>From c658310c0da3db301bfda9061ff9f5affc4e6111 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 29 Jun 2026 22:28:34 +0000
Subject: [PATCH 07/15] format
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index d2bde98dbcbaa..d6930235a387c 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -385,10 +385,14 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
- bool IsElementwiseExpand = RMWI->isElementwise() && TLI->shouldExpandAtomicRMWInIR(RMWI) == TargetLoweringBase::AtomicExpansionKind::Expand;
+ bool IsElementwiseExpand =
+ RMWI->isElementwise() &&
+ TLI->shouldExpandAtomicRMWInIR(RMWI) ==
+ TargetLoweringBase::AtomicExpansionKind::Expand;
if (!atomicSizeSupported(TLI, RMWI)) {
- // Elementwise expansion may split an atomicrmw into smaller, supported atomic sizes.
+ // Elementwise expansion may split an atomicrmw into smaller, supported
+ // atomic sizes.
if (!IsElementwiseExpand) {
expandAtomicRMWToLibcall(RMWI);
return true;
>From c38a65b04ad61fc6cbb1bd5ffe2c248858c1951f Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 30 Jun 2026 23:42:01 +0000
Subject: [PATCH 08/15] fix rebase conflict
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index df6fe672c9030..470fd778838d8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7340,10 +7340,10 @@ void NVPTXTargetLowering::ReplaceNodeResults(
}
}
-/// Returns how NVPTX should expand a scalar atomicrmw with the given op and
-/// value type.
+/// Returns how NVPTX should expand a scalar atomicrmw with the given
+/// instruction and value type.
static TargetLoweringBase::AtomicExpansionKind
-getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
+getScalarAtomicRMWExpansion(const AtomicRMWInst *AI, Type *Ty,
const NVPTXSubtarget &STI) {
using AtomicExpansionKind = TargetLoweringBase::AtomicExpansionKind;
@@ -7401,7 +7401,7 @@ getScalarAtomicRMWExpansion(AtomicRMWInst::BinOp Op, Type *Ty,
"integer.");
const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
- switch (Op) {
+ switch (AI->getOperation()) {
default:
return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Xchg:
@@ -7481,7 +7481,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// eventually bottoms out at the scalar base case, where this hook returns
// None for each scalar lane and the scalar `atom.*` instruction is
// preserved.
- if (getScalarAtomicRMWExpansion(AI->getOperation(), LaneTy, STI) ==
+ if (getScalarAtomicRMWExpansion(AI, LaneTy, STI) ==
AtomicExpansionKind::None)
return AtomicExpansionKind::Expand;
@@ -7497,8 +7497,7 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
return AtomicExpansionKind::Expand;
}
- return getScalarAtomicRMWExpansion(AI->getOperation(),
- AI->getValOperand()->getType(), STI);
+ return getScalarAtomicRMWExpansion(AI, AI->getValOperand()->getType(), STI);
}
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
>From f627a2dd23c9ad1621dcd7429f78297b97650003 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 1 Jul 2026 02:21:24 +0000
Subject: [PATCH 09/15] fix tests
---
.../NVPTX/atomicrmw-elementwise-sm60.ll | 3394 ++++++++---------
.../NVPTX/atomicrmw-elementwise-sm70.ll | 3394 ++++++++---------
.../NVPTX/atomicrmw-elementwise-sm90.ll | 944 +++--
.../NVPTX/expand-atomic-rmw-elementwise.ll | 33 +-
4 files changed, 3542 insertions(+), 4223 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index ab48ec6523719..4a7b2132cfb3c 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -9919,88 +9919,88 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
-define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
-; SM60-LABEL: fadd_acq_rel_v1f32_global_cta(
+define <1 x float> @fadd_acq_rel_v1f32_shared_cta(ptr addrspace(3) %addr, <1 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v1f32_shared_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<3>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_shared_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_shared_cta_param_1];
+; SM60-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
}
-define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
-; SM60-LABEL: fadd_acq_rel_v2f32_global_cta(
+define <2 x float> @fadd_acq_rel_v2f32_shared_cta(ptr addrspace(3) %addr, <2 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v2f32_shared_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<5>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_shared_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_shared_cta_param_1];
+; SM60-NEXT: atom.cta.shared.add.f32 %r3, [%rd1], %r1;
+; SM60-NEXT: atom.cta.shared.add.f32 %r4, [%rd1+4], %r2;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
}
-define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
-; SM60-LABEL: fadd_acq_rel_v4f32_global_cta(
+define <4 x float> @fadd_acq_rel_v4f32_shared_cta(ptr addrspace(3) %addr, <4 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v4f32_shared_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_shared_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM60-NEXT: atom.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM60-NEXT: atom.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_shared_cta_param_1];
+; SM60-NEXT: atom.cta.shared.add.f32 %r5, [%rd1], %r1;
+; SM60-NEXT: atom.cta.shared.add.f32 %r6, [%rd1+4], %r2;
+; SM60-NEXT: atom.cta.shared.add.f32 %r7, [%rd1+8], %r3;
+; SM60-NEXT: atom.cta.shared.add.f32 %r8, [%rd1+12], %r4;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
}
-define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
-; SM60-LABEL: fadd_acq_rel_v8f32_global_cta(
+define <8 x float> @fadd_acq_rel_v8f32_shared_cta(ptr addrspace(3) %addr, <8 x float> %val) {
+; SM60-LABEL: fadd_acq_rel_v8f32_shared_cta(
; SM60: {
; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_shared_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM60-NEXT: atom.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM60-NEXT: atom.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM60-NEXT: atom.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM60-NEXT: atom.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM60-NEXT: atom.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM60-NEXT: atom.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM60-NEXT: atom.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_shared_cta_param_1+16];
+; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_shared_cta_param_1];
+; SM60-NEXT: atom.cta.shared.add.f32 %r9, [%rd1], %r5;
+; SM60-NEXT: atom.cta.shared.add.f32 %r10, [%rd1+4], %r6;
+; SM60-NEXT: atom.cta.shared.add.f32 %r11, [%rd1+8], %r7;
+; SM60-NEXT: atom.cta.shared.add.f32 %r12, [%rd1+12], %r8;
+; SM60-NEXT: atom.cta.shared.add.f32 %r13, [%rd1+16], %r1;
+; SM60-NEXT: atom.cta.shared.add.f32 %r14, [%rd1+20], %r2;
+; SM60-NEXT: atom.cta.shared.add.f32 %r15, [%rd1+24], %r3;
+; SM60-NEXT: atom.cta.shared.add.f32 %r16, [%rd1+28], %r4;
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
}
@@ -10697,30 +10697,28 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM60-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<8>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
-; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM60-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NEXT: $L__BB256_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
-; SM60-NEXT: min.f32 %r3, %r8, %r1;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
-; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
-; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
-; SM60-NEXT: mov.b32 %r8, %r2;
-; SM60-NEXT: @%p5 bra $L__BB256_1;
+; SM60-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM60-NEXT: selp.f32 %r3, %r7, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r7, %r1;
+; SM60-NEXT: min.f32 %r4, %r7, %r1;
+; SM60-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
+; SM60-NEXT: setp.ne.b32 %p4, %r2, %r7;
+; SM60-NEXT: mov.b32 %r7, %r2;
+; SM60-NEXT: @%p4 bra $L__BB256_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
@@ -10732,8 +10730,8 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<10>;
-; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .pred %p<8>;
+; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -10741,44 +10739,40 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd2;
; SM60-NEXT: $L__BB257_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
-; SM60-NEXT: min.f32 %r3, %r13, %r1;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
-; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
-; SM60-NEXT: min.f32 %r8, %r14, %r2;
+; SM60-NEXT: setp.eq.b32 %p1, %r11, -2147483648;
+; SM60-NEXT: selp.f32 %r3, %r11, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r11, %r1;
+; SM60-NEXT: min.f32 %r4, %r11, %r1;
+; SM60-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r11, %r1;
+; SM60-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: setp.eq.b32 %p4, %r12, -2147483648;
+; SM60-NEXT: selp.f32 %r7, %r12, %r2, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r12, %r2;
+; SM60-NEXT: min.f32 %r8, %r12, %r2;
; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
-; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
-; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
-; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: setp.eq.f32 %p6, %r12, %r2;
+; SM60-NEXT: selp.f32 %r10, %r7, %r9, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB257_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB257_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10787,8 +10781,8 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM60-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<19>;
-; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .pred %p<15>;
+; SM60-NEXT: .reg .b32 %r<25>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -10796,81 +10790,73 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
; SM60-NEXT: $L__BB258_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
-; SM60-NEXT: min.f32 %r5, %r25, %r1;
-; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
-; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
-; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
-; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
-; SM60-NEXT: min.f32 %r10, %r26, %r2;
+; SM60-NEXT: setp.eq.b32 %p1, %r21, -2147483648;
+; SM60-NEXT: selp.f32 %r5, %r21, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r21, %r1;
+; SM60-NEXT: min.f32 %r6, %r21, %r1;
+; SM60-NEXT: selp.f32 %r7, 0f7FC00000, %r6, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r21, %r1;
+; SM60-NEXT: selp.f32 %r8, %r5, %r7, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM60-NEXT: setp.eq.b32 %p4, %r22, -2147483648;
+; SM60-NEXT: selp.f32 %r9, %r22, %r2, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r22, %r2;
+; SM60-NEXT: min.f32 %r10, %r22, %r2;
; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
-; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
-; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
-; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: setp.eq.f32 %p6, %r22, %r2;
+; SM60-NEXT: selp.f32 %r12, %r9, %r11, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB258_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB258_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM60-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd12;
; SM60-NEXT: $L__BB258_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
-; SM60-NEXT: min.f32 %r15, %r27, %r3;
-; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
-; SM60-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
-; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
-; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
-; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
-; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
-; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
-; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
-; SM60-NEXT: min.f32 %r20, %r28, %r4;
-; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
-; SM60-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
-; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
-; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
-; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
-; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
-; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: setp.eq.b32 %p8, %r23, -2147483648;
+; SM60-NEXT: selp.f32 %r13, %r23, %r3, %p8;
+; SM60-NEXT: setp.nan.f32 %p9, %r23, %r3;
+; SM60-NEXT: min.f32 %r14, %r23, %r3;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p9;
+; SM60-NEXT: setp.eq.f32 %p10, %r23, %r3;
+; SM60-NEXT: selp.f32 %r16, %r13, %r15, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM60-NEXT: setp.eq.b32 %p11, %r24, -2147483648;
+; SM60-NEXT: selp.f32 %r17, %r24, %r4, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r24, %r4;
+; SM60-NEXT: min.f32 %r18, %r24, %r4;
+; SM60-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r24, %r4;
+; SM60-NEXT: selp.f32 %r20, %r17, %r19, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r20;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r24;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB258_3;
+; SM60-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM60-NEXT: @%p14 bra $L__BB258_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10879,8 +10865,8 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM60-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<37>;
-; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .pred %p<29>;
+; SM60-NEXT: .reg .b32 %r<49>;
; SM60-NEXT: .reg .b64 %rd<42>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -10889,156 +10875,140 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: $L__BB259_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
-; SM60-NEXT: min.f32 %r9, %r49, %r5;
-; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
-; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
-; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
-; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
-; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
-; SM60-NEXT: min.f32 %r14, %r50, %r6;
+; SM60-NEXT: setp.eq.b32 %p1, %r41, -2147483648;
+; SM60-NEXT: selp.f32 %r9, %r41, %r5, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r41, %r5;
+; SM60-NEXT: min.f32 %r10, %r41, %r5;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r41, %r5;
+; SM60-NEXT: selp.f32 %r12, %r9, %r11, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r12;
+; SM60-NEXT: setp.eq.b32 %p4, %r42, -2147483648;
+; SM60-NEXT: selp.f32 %r13, %r42, %r6, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r42, %r6;
+; SM60-NEXT: min.f32 %r14, %r42, %r6;
; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
-; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
-; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
-; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: setp.eq.f32 %p6, %r42, %r6;
+; SM60-NEXT: selp.f32 %r16, %r13, %r15, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r42;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB259_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB259_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
-; SM60-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd12;
; SM60-NEXT: $L__BB259_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
-; SM60-NEXT: min.f32 %r19, %r51, %r7;
-; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
-; SM60-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
-; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
-; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
-; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
-; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
-; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
-; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
-; SM60-NEXT: min.f32 %r24, %r52, %r8;
-; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
-; SM60-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
-; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
-; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
-; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
-; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
-; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: setp.eq.b32 %p8, %r43, -2147483648;
+; SM60-NEXT: selp.f32 %r17, %r43, %r7, %p8;
+; SM60-NEXT: setp.nan.f32 %p9, %r43, %r7;
+; SM60-NEXT: min.f32 %r18, %r43, %r7;
+; SM60-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p9;
+; SM60-NEXT: setp.eq.f32 %p10, %r43, %r7;
+; SM60-NEXT: selp.f32 %r20, %r17, %r19, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r20;
+; SM60-NEXT: setp.eq.b32 %p11, %r44, -2147483648;
+; SM60-NEXT: selp.f32 %r21, %r44, %r8, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r44, %r8;
+; SM60-NEXT: min.f32 %r22, %r44, %r8;
+; SM60-NEXT: selp.f32 %r23, 0f7FC00000, %r22, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r44, %r8;
+; SM60-NEXT: selp.f32 %r24, %r21, %r23, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r44;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB259_3;
+; SM60-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM60-NEXT: @%p14 bra $L__BB259_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
-; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
-; SM60-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd22;
; SM60-NEXT: $L__BB259_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
-; SM60-NEXT: min.f32 %r29, %r53, %r1;
-; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
-; SM60-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
-; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
-; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
-; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
-; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
-; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
-; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
-; SM60-NEXT: min.f32 %r34, %r54, %r2;
-; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
-; SM60-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
-; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
-; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
-; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
-; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
-; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: setp.eq.b32 %p15, %r45, -2147483648;
+; SM60-NEXT: selp.f32 %r25, %r45, %r1, %p15;
+; SM60-NEXT: setp.nan.f32 %p16, %r45, %r1;
+; SM60-NEXT: min.f32 %r26, %r45, %r1;
+; SM60-NEXT: selp.f32 %r27, 0f7FC00000, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r45, %r1;
+; SM60-NEXT: selp.f32 %r28, %r25, %r27, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r28;
+; SM60-NEXT: setp.eq.b32 %p18, %r46, -2147483648;
+; SM60-NEXT: selp.f32 %r29, %r46, %r2, %p18;
+; SM60-NEXT: setp.nan.f32 %p19, %r46, %r2;
+; SM60-NEXT: min.f32 %r30, %r46, %r2;
+; SM60-NEXT: selp.f32 %r31, 0f7FC00000, %r30, %p19;
+; SM60-NEXT: setp.eq.f32 %p20, %r46, %r2;
+; SM60-NEXT: selp.f32 %r32, %r29, %r31, %p20;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r32;
; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
-; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r46;
; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
-; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
-; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM60-NEXT: @%p27 bra $L__BB259_5;
+; SM60-NEXT: setp.ne.b64 %p21, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd31;
+; SM60-NEXT: @%p21 bra $L__BB259_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
-; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
-; SM60-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
-; SM60-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r47, %rd32;
; SM60-NEXT: $L__BB259_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
-; SM60-NEXT: min.f32 %r39, %r55, %r3;
-; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
-; SM60-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
-; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
-; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
-; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
-; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
-; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
-; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
-; SM60-NEXT: min.f32 %r44, %r56, %r4;
-; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
-; SM60-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
-; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
-; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
-; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
-; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
-; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: setp.eq.b32 %p22, %r47, -2147483648;
+; SM60-NEXT: selp.f32 %r33, %r47, %r3, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r47, %r3;
+; SM60-NEXT: min.f32 %r34, %r47, %r3;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.f32 %p24, %r47, %r3;
+; SM60-NEXT: selp.f32 %r36, %r33, %r35, %p24;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r36;
+; SM60-NEXT: setp.eq.b32 %p25, %r48, -2147483648;
+; SM60-NEXT: selp.f32 %r37, %r48, %r4, %p25;
+; SM60-NEXT: setp.nan.f32 %p26, %r48, %r4;
+; SM60-NEXT: min.f32 %r38, %r48, %r4;
+; SM60-NEXT: selp.f32 %r39, 0f7FC00000, %r38, %p26;
+; SM60-NEXT: setp.eq.f32 %p27, %r48, %r4;
+; SM60-NEXT: selp.f32 %r40, %r37, %r39, %p27;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r40;
; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
-; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r47;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r48;
; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
-; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
-; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM60-NEXT: @%p36 bra $L__BB259_7;
+; SM60-NEXT: setp.ne.b64 %p28, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r47, %rd41;
+; SM60-NEXT: @%p28 bra $L__BB259_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11047,30 +11017,28 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM60-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b32 %r<9>;
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b32 %r<8>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
-; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM60-NEXT: ld.volatile.global.b32 %r7, [%rd1];
; SM60-NEXT: $L__BB260_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r8, %r1;
-; SM60-NEXT: max.f32 %r3, %r8, %r1;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r8, 0;
-; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
-; SM60-NEXT: setp.ne.b32 %p5, %r2, %r8;
-; SM60-NEXT: mov.b32 %r8, %r2;
-; SM60-NEXT: @%p5 bra $L__BB260_1;
+; SM60-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM60-NEXT: selp.f32 %r3, %r7, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r7, %r1;
+; SM60-NEXT: max.f32 %r4, %r7, %r1;
+; SM60-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM60-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
+; SM60-NEXT: setp.ne.b32 %p4, %r2, %r7;
+; SM60-NEXT: mov.b32 %r7, %r2;
+; SM60-NEXT: @%p4 bra $L__BB260_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
@@ -11082,8 +11050,8 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<10>;
-; SM60-NEXT: .reg .b32 %r<15>;
+; SM60-NEXT: .reg .pred %p<8>;
+; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -11091,44 +11059,40 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
-; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd2;
; SM60-NEXT: $L__BB261_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r13, %r1;
-; SM60-NEXT: max.f32 %r3, %r13, %r1;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r13, 0;
-; SM60-NEXT: selp.f32 %r5, %r13, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: setp.nan.f32 %p5, %r14, %r2;
-; SM60-NEXT: max.f32 %r8, %r14, %r2;
+; SM60-NEXT: setp.eq.b32 %p1, %r11, 0;
+; SM60-NEXT: selp.f32 %r3, %r11, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r11, %r1;
+; SM60-NEXT: max.f32 %r4, %r11, %r1;
+; SM60-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r11, %r1;
+; SM60-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM60-NEXT: setp.eq.b32 %p4, %r12, 0;
+; SM60-NEXT: selp.f32 %r7, %r12, %r2, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r12, %r2;
+; SM60-NEXT: max.f32 %r8, %r12, %r2;
; SM60-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r14, 0;
-; SM60-NEXT: selp.f32 %r10, %r14, %r9, %p6;
-; SM60-NEXT: selp.f32 %r11, %r2, %r10, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
-; SM60-NEXT: selp.f32 %r12, %r11, %r9, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM60-NEXT: setp.eq.f32 %p6, %r12, %r2;
+; SM60-NEXT: selp.f32 %r10, %r7, %r9, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r13;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r12;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB261_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB261_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -11137,8 +11101,8 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM60-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<19>;
-; SM60-NEXT: .reg .b32 %r<29>;
+; SM60-NEXT: .reg .pred %p<15>;
+; SM60-NEXT: .reg .b32 %r<25>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -11146,81 +11110,73 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r1, 0;
-; SM60-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
; SM60-NEXT: $L__BB262_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r25, %r1;
-; SM60-NEXT: max.f32 %r5, %r25, %r1;
-; SM60-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r25, 0;
-; SM60-NEXT: selp.f32 %r7, %r25, %r6, %p2;
-; SM60-NEXT: selp.f32 %r8, %r1, %r7, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
-; SM60-NEXT: selp.f32 %r9, %r8, %r6, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: setp.nan.f32 %p5, %r26, %r2;
-; SM60-NEXT: max.f32 %r10, %r26, %r2;
+; SM60-NEXT: setp.eq.b32 %p1, %r21, 0;
+; SM60-NEXT: selp.f32 %r5, %r21, %r1, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r21, %r1;
+; SM60-NEXT: max.f32 %r6, %r21, %r1;
+; SM60-NEXT: selp.f32 %r7, 0f7FC00000, %r6, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r21, %r1;
+; SM60-NEXT: selp.f32 %r8, %r5, %r7, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM60-NEXT: setp.eq.b32 %p4, %r22, 0;
+; SM60-NEXT: selp.f32 %r9, %r22, %r2, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r22, %r2;
+; SM60-NEXT: max.f32 %r10, %r22, %r2;
; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r26, 0;
-; SM60-NEXT: selp.f32 %r12, %r26, %r11, %p6;
-; SM60-NEXT: selp.f32 %r13, %r2, %r12, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
-; SM60-NEXT: selp.f32 %r14, %r13, %r11, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM60-NEXT: setp.eq.f32 %p6, %r22, %r2;
+; SM60-NEXT: selp.f32 %r12, %r9, %r11, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r12;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r25;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB262_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB262_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM60-NEXT: setp.eq.b32 %p12, %r3, 0;
-; SM60-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd12;
; SM60-NEXT: $L__BB262_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p10, %r27, %r3;
-; SM60-NEXT: max.f32 %r15, %r27, %r3;
-; SM60-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
-; SM60-NEXT: setp.eq.b32 %p11, %r27, 0;
-; SM60-NEXT: selp.f32 %r17, %r27, %r16, %p11;
-; SM60-NEXT: selp.f32 %r18, %r3, %r17, %p12;
-; SM60-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
-; SM60-NEXT: selp.f32 %r19, %r18, %r16, %p13;
-; SM60-NEXT: cvt.u64.u32 %rd13, %r19;
-; SM60-NEXT: setp.nan.f32 %p14, %r28, %r4;
-; SM60-NEXT: max.f32 %r20, %r28, %r4;
-; SM60-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
-; SM60-NEXT: setp.eq.b32 %p15, %r28, 0;
-; SM60-NEXT: selp.f32 %r22, %r28, %r21, %p15;
-; SM60-NEXT: selp.f32 %r23, %r4, %r22, %p16;
-; SM60-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
-; SM60-NEXT: selp.f32 %r24, %r23, %r21, %p17;
-; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM60-NEXT: setp.eq.b32 %p8, %r23, 0;
+; SM60-NEXT: selp.f32 %r13, %r23, %r3, %p8;
+; SM60-NEXT: setp.nan.f32 %p9, %r23, %r3;
+; SM60-NEXT: max.f32 %r14, %r23, %r3;
+; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p9;
+; SM60-NEXT: setp.eq.f32 %p10, %r23, %r3;
+; SM60-NEXT: selp.f32 %r16, %r13, %r15, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM60-NEXT: setp.eq.b32 %p11, %r24, 0;
+; SM60-NEXT: selp.f32 %r17, %r24, %r4, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r24, %r4;
+; SM60-NEXT: max.f32 %r18, %r24, %r4;
+; SM60-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r24, %r4;
+; SM60-NEXT: selp.f32 %r20, %r17, %r19, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r20;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r27;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r24;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB262_3;
+; SM60-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM60-NEXT: @%p14 bra $L__BB262_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -11229,8 +11185,8 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM60-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<37>;
-; SM60-NEXT: .reg .b32 %r<57>;
+; SM60-NEXT: .reg .pred %p<29>;
+; SM60-NEXT: .reg .b32 %r<49>;
; SM60-NEXT: .reg .b64 %rd<42>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -11239,156 +11195,140 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM60-NEXT: setp.eq.b32 %p3, %r5, 0;
-; SM60-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: $L__BB263_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r49, %r5;
-; SM60-NEXT: max.f32 %r9, %r49, %r5;
-; SM60-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r49, 0;
-; SM60-NEXT: selp.f32 %r11, %r49, %r10, %p2;
-; SM60-NEXT: selp.f32 %r12, %r5, %r11, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
-; SM60-NEXT: selp.f32 %r13, %r12, %r10, %p4;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
-; SM60-NEXT: setp.nan.f32 %p5, %r50, %r6;
-; SM60-NEXT: max.f32 %r14, %r50, %r6;
+; SM60-NEXT: setp.eq.b32 %p1, %r41, 0;
+; SM60-NEXT: selp.f32 %r9, %r41, %r5, %p1;
+; SM60-NEXT: setp.nan.f32 %p2, %r41, %r5;
+; SM60-NEXT: max.f32 %r10, %r41, %r5;
+; SM60-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p2;
+; SM60-NEXT: setp.eq.f32 %p3, %r41, %r5;
+; SM60-NEXT: selp.f32 %r12, %r9, %r11, %p3;
+; SM60-NEXT: cvt.u64.u32 %rd3, %r12;
+; SM60-NEXT: setp.eq.b32 %p4, %r42, 0;
+; SM60-NEXT: selp.f32 %r13, %r42, %r6, %p4;
+; SM60-NEXT: setp.nan.f32 %p5, %r42, %r6;
+; SM60-NEXT: max.f32 %r14, %r42, %r6;
; SM60-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
-; SM60-NEXT: setp.eq.b32 %p6, %r50, 0;
-; SM60-NEXT: selp.f32 %r16, %r50, %r15, %p6;
-; SM60-NEXT: selp.f32 %r17, %r6, %r16, %p7;
-; SM60-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
-; SM60-NEXT: selp.f32 %r18, %r17, %r15, %p8;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM60-NEXT: setp.eq.f32 %p6, %r42, %r6;
+; SM60-NEXT: selp.f32 %r16, %r13, %r15, %p6;
+; SM60-NEXT: cvt.u64.u32 %rd4, %r16;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r49;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r42;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM60-NEXT: @%p9 bra $L__BB263_1;
+; SM60-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM60-NEXT: @%p7 bra $L__BB263_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r51, %rd12;
-; SM60-NEXT: setp.eq.b32 %p12, %r7, 0;
-; SM60-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd12;
; SM60-NEXT: $L__BB263_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p10, %r51, %r7;
-; SM60-NEXT: max.f32 %r19, %r51, %r7;
-; SM60-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
-; SM60-NEXT: setp.eq.b32 %p11, %r51, 0;
-; SM60-NEXT: selp.f32 %r21, %r51, %r20, %p11;
-; SM60-NEXT: selp.f32 %r22, %r7, %r21, %p12;
-; SM60-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
-; SM60-NEXT: selp.f32 %r23, %r22, %r20, %p13;
-; SM60-NEXT: cvt.u64.u32 %rd13, %r23;
-; SM60-NEXT: setp.nan.f32 %p14, %r52, %r8;
-; SM60-NEXT: max.f32 %r24, %r52, %r8;
-; SM60-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
-; SM60-NEXT: setp.eq.b32 %p15, %r52, 0;
-; SM60-NEXT: selp.f32 %r26, %r52, %r25, %p15;
-; SM60-NEXT: selp.f32 %r27, %r8, %r26, %p16;
-; SM60-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
-; SM60-NEXT: selp.f32 %r28, %r27, %r25, %p17;
-; SM60-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM60-NEXT: setp.eq.b32 %p8, %r43, 0;
+; SM60-NEXT: selp.f32 %r17, %r43, %r7, %p8;
+; SM60-NEXT: setp.nan.f32 %p9, %r43, %r7;
+; SM60-NEXT: max.f32 %r18, %r43, %r7;
+; SM60-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p9;
+; SM60-NEXT: setp.eq.f32 %p10, %r43, %r7;
+; SM60-NEXT: selp.f32 %r20, %r17, %r19, %p10;
+; SM60-NEXT: cvt.u64.u32 %rd13, %r20;
+; SM60-NEXT: setp.eq.b32 %p11, %r44, 0;
+; SM60-NEXT: selp.f32 %r21, %r44, %r8, %p11;
+; SM60-NEXT: setp.nan.f32 %p12, %r44, %r8;
+; SM60-NEXT: max.f32 %r22, %r44, %r8;
+; SM60-NEXT: selp.f32 %r23, 0f7FC00000, %r22, %p12;
+; SM60-NEXT: setp.eq.f32 %p13, %r44, %r8;
+; SM60-NEXT: selp.f32 %r24, %r21, %r23, %p13;
+; SM60-NEXT: cvt.u64.u32 %rd14, %r24;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r51;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r44;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM60-NEXT: @%p18 bra $L__BB263_3;
+; SM60-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM60-NEXT: @%p14 bra $L__BB263_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd22, [%rd1+16];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
-; SM60-NEXT: cvt.u32.u64 %r53, %rd22;
-; SM60-NEXT: setp.eq.b32 %p21, %r1, 0;
-; SM60-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd22;
; SM60-NEXT: $L__BB263_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p19, %r53, %r1;
-; SM60-NEXT: max.f32 %r29, %r53, %r1;
-; SM60-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
-; SM60-NEXT: setp.eq.b32 %p20, %r53, 0;
-; SM60-NEXT: selp.f32 %r31, %r53, %r30, %p20;
-; SM60-NEXT: selp.f32 %r32, %r1, %r31, %p21;
-; SM60-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
-; SM60-NEXT: selp.f32 %r33, %r32, %r30, %p22;
-; SM60-NEXT: cvt.u64.u32 %rd23, %r33;
-; SM60-NEXT: setp.nan.f32 %p23, %r54, %r2;
-; SM60-NEXT: max.f32 %r34, %r54, %r2;
-; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
-; SM60-NEXT: setp.eq.b32 %p24, %r54, 0;
-; SM60-NEXT: selp.f32 %r36, %r54, %r35, %p24;
-; SM60-NEXT: selp.f32 %r37, %r2, %r36, %p25;
-; SM60-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
-; SM60-NEXT: selp.f32 %r38, %r37, %r35, %p26;
-; SM60-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM60-NEXT: setp.eq.b32 %p15, %r45, 0;
+; SM60-NEXT: selp.f32 %r25, %r45, %r1, %p15;
+; SM60-NEXT: setp.nan.f32 %p16, %r45, %r1;
+; SM60-NEXT: max.f32 %r26, %r45, %r1;
+; SM60-NEXT: selp.f32 %r27, 0f7FC00000, %r26, %p16;
+; SM60-NEXT: setp.eq.f32 %p17, %r45, %r1;
+; SM60-NEXT: selp.f32 %r28, %r25, %r27, %p17;
+; SM60-NEXT: cvt.u64.u32 %rd23, %r28;
+; SM60-NEXT: setp.eq.b32 %p18, %r46, 0;
+; SM60-NEXT: selp.f32 %r29, %r46, %r2, %p18;
+; SM60-NEXT: setp.nan.f32 %p19, %r46, %r2;
+; SM60-NEXT: max.f32 %r30, %r46, %r2;
+; SM60-NEXT: selp.f32 %r31, 0f7FC00000, %r30, %p19;
+; SM60-NEXT: setp.eq.f32 %p20, %r46, %r2;
+; SM60-NEXT: selp.f32 %r32, %r29, %r31, %p20;
+; SM60-NEXT: cvt.u64.u32 %rd24, %r32;
; SM60-NEXT: shl.b64 %rd25, %rd24, 32;
; SM60-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM60-NEXT: cvt.u64.u32 %rd27, %r53;
-; SM60-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM60-NEXT: cvt.u64.u32 %rd27, %r45;
+; SM60-NEXT: cvt.u64.u32 %rd28, %r46;
; SM60-NEXT: shl.b64 %rd29, %rd28, 32;
; SM60-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
-; SM60-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
-; SM60-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM60-NEXT: @%p27 bra $L__BB263_5;
+; SM60-NEXT: setp.ne.b64 %p21, %rd31, %rd30;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
+; SM60-NEXT: cvt.u32.u64 %r45, %rd31;
+; SM60-NEXT: @%p21 bra $L__BB263_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd1+24];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
-; SM60-NEXT: cvt.u32.u64 %r55, %rd32;
-; SM60-NEXT: setp.eq.b32 %p30, %r3, 0;
-; SM60-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
+; SM60-NEXT: cvt.u32.u64 %r47, %rd32;
; SM60-NEXT: $L__BB263_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p28, %r55, %r3;
-; SM60-NEXT: max.f32 %r39, %r55, %r3;
-; SM60-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
-; SM60-NEXT: setp.eq.b32 %p29, %r55, 0;
-; SM60-NEXT: selp.f32 %r41, %r55, %r40, %p29;
-; SM60-NEXT: selp.f32 %r42, %r3, %r41, %p30;
-; SM60-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
-; SM60-NEXT: selp.f32 %r43, %r42, %r40, %p31;
-; SM60-NEXT: cvt.u64.u32 %rd33, %r43;
-; SM60-NEXT: setp.nan.f32 %p32, %r56, %r4;
-; SM60-NEXT: max.f32 %r44, %r56, %r4;
-; SM60-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
-; SM60-NEXT: setp.eq.b32 %p33, %r56, 0;
-; SM60-NEXT: selp.f32 %r46, %r56, %r45, %p33;
-; SM60-NEXT: selp.f32 %r47, %r4, %r46, %p34;
-; SM60-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
-; SM60-NEXT: selp.f32 %r48, %r47, %r45, %p35;
-; SM60-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM60-NEXT: setp.eq.b32 %p22, %r47, 0;
+; SM60-NEXT: selp.f32 %r33, %r47, %r3, %p22;
+; SM60-NEXT: setp.nan.f32 %p23, %r47, %r3;
+; SM60-NEXT: max.f32 %r34, %r47, %r3;
+; SM60-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM60-NEXT: setp.eq.f32 %p24, %r47, %r3;
+; SM60-NEXT: selp.f32 %r36, %r33, %r35, %p24;
+; SM60-NEXT: cvt.u64.u32 %rd33, %r36;
+; SM60-NEXT: setp.eq.b32 %p25, %r48, 0;
+; SM60-NEXT: selp.f32 %r37, %r48, %r4, %p25;
+; SM60-NEXT: setp.nan.f32 %p26, %r48, %r4;
+; SM60-NEXT: max.f32 %r38, %r48, %r4;
+; SM60-NEXT: selp.f32 %r39, 0f7FC00000, %r38, %p26;
+; SM60-NEXT: setp.eq.f32 %p27, %r48, %r4;
+; SM60-NEXT: selp.f32 %r40, %r37, %r39, %p27;
+; SM60-NEXT: cvt.u64.u32 %rd34, %r40;
; SM60-NEXT: shl.b64 %rd35, %rd34, 32;
; SM60-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM60-NEXT: cvt.u64.u32 %rd37, %r55;
-; SM60-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM60-NEXT: cvt.u64.u32 %rd37, %r47;
+; SM60-NEXT: cvt.u64.u32 %rd38, %r48;
; SM60-NEXT: shl.b64 %rd39, %rd38, 32;
; SM60-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM60-NEXT: atom.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
-; SM60-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
-; SM60-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM60-NEXT: @%p36 bra $L__BB263_7;
+; SM60-NEXT: setp.ne.b64 %p28, %rd41, %rd40;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
+; SM60-NEXT: cvt.u32.u64 %r47, %rd41;
+; SM60-NEXT: @%p28 bra $L__BB263_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -12117,29 +12057,27 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM60-LABEL: fminimum_acq_rel_v1f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<9>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
-; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd3];
; SM60-NEXT: $L__BB280_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM60-NEXT: min.f64 %rd4, %rd9, %rd1;
-; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM60-NEXT: mov.b64 %rd9, %rd2;
-; SM60-NEXT: @%p5 bra $L__BB280_1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM60-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM60-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd2;
+; SM60-NEXT: @%p4 bra $L__BB280_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12151,46 +12089,42 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<11>;
-; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<16>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
-; SM60-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd5];
; SM60-NEXT: $L__BB281_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
-; SM60-NEXT: min.f64 %rd6, %rd16, %rd1;
-; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
-; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
-; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
-; SM60-NEXT: mov.b64 %rd16, %rd3;
-; SM60-NEXT: @%p5 bra $L__BB281_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM60-NEXT: setp.eq.b64 %p1, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM60-NEXT: min.f64 %rd7, %rd14, %rd1;
+; SM60-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
+; SM60-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM60-NEXT: mov.b64 %rd14, %rd3;
+; SM60-NEXT: @%p4 bra $L__BB281_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd5+8];
; SM60-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
-; SM60-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: setp.eq.b64 %p5, %rd15, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM60-NEXT: min.f64 %rd11, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
-; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
-; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
-; SM60-NEXT: mov.b64 %rd17, %rd4;
-; SM60-NEXT: @%p10 bra $L__BB281_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
+; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
+; SM60-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
+; SM60-NEXT: mov.b64 %rd15, %rd4;
+; SM60-NEXT: @%p8 bra $L__BB281_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -12202,85 +12136,77 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM60-LABEL: fminimum_acq_rel_v4f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<21>;
-; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<30>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
-; SM60-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd9];
; SM60-NEXT: $L__BB282_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd30;
-; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
-; SM60-NEXT: min.f64 %rd10, %rd5, %rd3;
-; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
-; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
-; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM60-NEXT: @%p5 bra $L__BB282_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM60-NEXT: mov.b64 %rd5, %rd26;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM60-NEXT: min.f64 %rd11, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
+; SM60-NEXT: @%p4 bra $L__BB282_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd9+8];
; SM60-NEXT: $L__BB282_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: mov.b64 %rd6, %rd27;
+; SM60-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
; SM60-NEXT: min.f64 %rd15, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
-; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
-; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM60-NEXT: @%p10 bra $L__BB282_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
+; SM60-NEXT: @%p8 bra $L__BB282_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
-; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
-; SM60-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd9+16];
; SM60-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
-; SM60-NEXT: min.f64 %rd20, %rd32, %rd1;
-; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
-; SM60-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
-; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
-; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
-; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
-; SM60-NEXT: mov.b64 %rd32, %rd7;
-; SM60-NEXT: @%p15 bra $L__BB282_5;
+; SM60-NEXT: setp.eq.b64 %p9, %rd28, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM60-NEXT: min.f64 %rd19, %rd28, %rd1;
+; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
+; SM60-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
+; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM60-NEXT: mov.b64 %rd28, %rd7;
+; SM60-NEXT: @%p12 bra $L__BB282_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
-; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
-; SM60-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd9+24];
; SM60-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
-; SM60-NEXT: min.f64 %rd25, %rd33, %rd2;
-; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
-; SM60-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
-; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
-; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
-; SM60-NEXT: mov.b64 %rd33, %rd8;
-; SM60-NEXT: @%p20 bra $L__BB282_7;
+; SM60-NEXT: setp.eq.b64 %p13, %rd29, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM60-NEXT: min.f64 %rd23, %rd29, %rd2;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
+; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
+; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
+; SM60-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
+; SM60-NEXT: mov.b64 %rd29, %rd8;
+; SM60-NEXT: @%p16 bra $L__BB282_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12289,8 +12215,8 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM60-LABEL: fminimum_acq_rel_v8f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<41>;
-; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-NEXT: .reg .pred %p<33>;
+; SM60-NEXT: .reg .b64 %rd<58>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
@@ -12299,147 +12225,131 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
-; SM60-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd50, [%rd17];
; SM60-NEXT: $L__BB283_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd58;
-; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
-; SM60-NEXT: min.f64 %rd18, %rd7, %rd5;
-; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
-; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
-; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM60-NEXT: @%p5 bra $L__BB283_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM60-NEXT: mov.b64 %rd7, %rd50;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM60-NEXT: min.f64 %rd19, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
+; SM60-NEXT: @%p4 bra $L__BB283_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd51, [%rd17+8];
; SM60-NEXT: $L__BB283_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: mov.b64 %rd8, %rd51;
+; SM60-NEXT: setp.eq.b64 %p5, %rd8, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
; SM60-NEXT: min.f64 %rd23, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
-; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
-; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM60-NEXT: @%p10 bra $L__BB283_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB283_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
-; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
-; SM60-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd52, [%rd17+16];
; SM60-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd60;
-; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
-; SM60-NEXT: min.f64 %rd28, %rd9, %rd3;
-; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
-; SM60-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
-; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
-; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
-; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM60-NEXT: @%p15 bra $L__BB283_5;
+; SM60-NEXT: mov.b64 %rd9, %rd52;
+; SM60-NEXT: setp.eq.b64 %p9, %rd9, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM60-NEXT: min.f64 %rd27, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
+; SM60-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
+; SM60-NEXT: @%p12 bra $L__BB283_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
-; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
-; SM60-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd53, [%rd17+24];
; SM60-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd10, %rd61;
-; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
-; SM60-NEXT: min.f64 %rd33, %rd10, %rd4;
-; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
-; SM60-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
-; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
-; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM60-NEXT: @%p20 bra $L__BB283_7;
+; SM60-NEXT: mov.b64 %rd10, %rd53;
+; SM60-NEXT: setp.eq.b64 %p13, %rd10, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM60-NEXT: min.f64 %rd31, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
+; SM60-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
+; SM60-NEXT: @%p16 bra $L__BB283_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
-; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
-; SM60-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd54, [%rd17+32];
; SM60-NEXT: $L__BB283_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd62;
-; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
-; SM60-NEXT: min.f64 %rd38, %rd13, %rd1;
-; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
-; SM60-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
-; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
-; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
-; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM60-NEXT: @%p25 bra $L__BB283_9;
+; SM60-NEXT: mov.b64 %rd13, %rd54;
+; SM60-NEXT: setp.eq.b64 %p17, %rd13, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM60-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM60-NEXT: min.f64 %rd35, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
+; SM60-NEXT: @%p20 bra $L__BB283_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
-; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
-; SM60-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd55, [%rd17+40];
; SM60-NEXT: $L__BB283_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd14, %rd63;
-; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
-; SM60-NEXT: min.f64 %rd43, %rd14, %rd2;
-; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM60-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
-; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
-; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
-; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM60-NEXT: @%p30 bra $L__BB283_11;
+; SM60-NEXT: mov.b64 %rd14, %rd55;
+; SM60-NEXT: setp.eq.b64 %p21, %rd14, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM60-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM60-NEXT: min.f64 %rd39, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
+; SM60-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
+; SM60-NEXT: @%p24 bra $L__BB283_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
-; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
-; SM60-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd56, [%rd17+48];
; SM60-NEXT: $L__BB283_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
-; SM60-NEXT: min.f64 %rd48, %rd64, %rd11;
-; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
-; SM60-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
-; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
-; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
-; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
-; SM60-NEXT: mov.b64 %rd64, %rd15;
-; SM60-NEXT: @%p35 bra $L__BB283_13;
+; SM60-NEXT: setp.eq.b64 %p25, %rd56, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM60-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM60-NEXT: min.f64 %rd43, %rd56, %rd11;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
+; SM60-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM60-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM60-NEXT: mov.b64 %rd56, %rd15;
+; SM60-NEXT: @%p28 bra $L__BB283_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
-; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
-; SM60-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM60-NEXT: ld.volatile.global.b64 %rd57, [%rd17+56];
; SM60-NEXT: $L__BB283_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
-; SM60-NEXT: min.f64 %rd53, %rd65, %rd12;
-; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
-; SM60-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
-; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
-; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
-; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
-; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
-; SM60-NEXT: mov.b64 %rd65, %rd16;
-; SM60-NEXT: @%p40 bra $L__BB283_15;
+; SM60-NEXT: setp.eq.b64 %p29, %rd57, -9223372036854775808;
+; SM60-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM60-NEXT: min.f64 %rd47, %rd57, %rd12;
+; SM60-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
+; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
+; SM60-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
+; SM60-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
+; SM60-NEXT: mov.b64 %rd57, %rd16;
+; SM60-NEXT: @%p32 bra $L__BB283_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12448,29 +12358,27 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM60-LABEL: fmaximum_acq_rel_v1f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b64 %rd<10>;
+; SM60-NEXT: .reg .pred %p<5>;
+; SM60-NEXT: .reg .b64 %rd<9>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd9, [%rd3];
-; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd8, [%rd3];
; SM60-NEXT: $L__BB284_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM60-NEXT: max.f64 %rd4, %rd9, %rd1;
-; SM60-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd9, 0;
-; SM60-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM60-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM60-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM60-NEXT: mov.b64 %rd9, %rd2;
-; SM60-NEXT: @%p5 bra $L__BB284_1;
+; SM60-NEXT: setp.eq.b64 %p1, %rd8, 0;
+; SM60-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM60-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM60-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM60-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM60-NEXT: mov.b64 %rd8, %rd2;
+; SM60-NEXT: @%p4 bra $L__BB284_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12482,46 +12390,42 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<11>;
-; SM60-NEXT: .reg .b64 %rd<18>;
+; SM60-NEXT: .reg .pred %p<9>;
+; SM60-NEXT: .reg .b64 %rd<16>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd16, [%rd5];
-; SM60-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd14, [%rd5];
; SM60-NEXT: $L__BB285_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
-; SM60-NEXT: max.f64 %rd6, %rd16, %rd1;
-; SM60-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd16, 0;
-; SM60-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
-; SM60-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
-; SM60-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
-; SM60-NEXT: mov.b64 %rd16, %rd3;
-; SM60-NEXT: @%p5 bra $L__BB285_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd17, [%rd5+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM60-NEXT: setp.eq.b64 %p1, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM60-NEXT: max.f64 %rd7, %rd14, %rd1;
+; SM60-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
+; SM60-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM60-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM60-NEXT: mov.b64 %rd14, %rd3;
+; SM60-NEXT: @%p4 bra $L__BB285_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd15, [%rd5+8];
; SM60-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
-; SM60-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM60-NEXT: setp.eq.b64 %p5, %rd15, 0;
+; SM60-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM60-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM60-NEXT: max.f64 %rd11, %rd15, %rd2;
; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd17, 0;
-; SM60-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
-; SM60-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
-; SM60-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
-; SM60-NEXT: mov.b64 %rd17, %rd4;
-; SM60-NEXT: @%p10 bra $L__BB285_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
+; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
+; SM60-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
+; SM60-NEXT: mov.b64 %rd15, %rd4;
+; SM60-NEXT: @%p8 bra $L__BB285_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -12533,85 +12437,77 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM60-LABEL: fmaximum_acq_rel_v4f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<21>;
-; SM60-NEXT: .reg .b64 %rd<34>;
+; SM60-NEXT: .reg .pred %p<17>;
+; SM60-NEXT: .reg .b64 %rd<30>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd30, [%rd9];
-; SM60-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd26, [%rd9];
; SM60-NEXT: $L__BB286_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd5, %rd30;
-; SM60-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
-; SM60-NEXT: max.f64 %rd10, %rd5, %rd3;
-; SM60-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd5, 0;
-; SM60-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
-; SM60-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
-; SM60-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM60-NEXT: @%p5 bra $L__BB286_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd31, [%rd9+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM60-NEXT: mov.b64 %rd5, %rd26;
+; SM60-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM60-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM60-NEXT: max.f64 %rd11, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
+; SM60-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM60-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
+; SM60-NEXT: @%p4 bra $L__BB286_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd27, [%rd9+8];
; SM60-NEXT: $L__BB286_3: // %atomicrmw.start5
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd6, %rd31;
+; SM60-NEXT: mov.b64 %rd6, %rd27;
+; SM60-NEXT: setp.eq.b64 %p5, %rd6, 0;
+; SM60-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
; SM60-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
; SM60-NEXT: max.f64 %rd15, %rd6, %rd4;
; SM60-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd6, 0;
-; SM60-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
-; SM60-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
-; SM60-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM60-NEXT: @%p10 bra $L__BB286_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
+; SM60-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM60-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
+; SM60-NEXT: @%p8 bra $L__BB286_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end4
-; SM60-NEXT: ld.volatile.global.b64 %rd32, [%rd9+16];
-; SM60-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd28, [%rd9+16];
; SM60-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
-; SM60-NEXT: max.f64 %rd20, %rd32, %rd1;
-; SM60-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
-; SM60-NEXT: setp.eq.b64 %p12, %rd32, 0;
-; SM60-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
-; SM60-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
-; SM60-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
-; SM60-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
-; SM60-NEXT: mov.b64 %rd32, %rd7;
-; SM60-NEXT: @%p15 bra $L__BB286_5;
+; SM60-NEXT: setp.eq.b64 %p9, %rd28, 0;
+; SM60-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM60-NEXT: max.f64 %rd19, %rd28, %rd1;
+; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
+; SM60-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
+; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM60-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM60-NEXT: mov.b64 %rd28, %rd7;
+; SM60-NEXT: @%p12 bra $L__BB286_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end14
-; SM60-NEXT: ld.volatile.global.b64 %rd33, [%rd9+24];
-; SM60-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd29, [%rd9+24];
; SM60-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
-; SM60-NEXT: max.f64 %rd25, %rd33, %rd2;
-; SM60-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
-; SM60-NEXT: setp.eq.b64 %p17, %rd33, 0;
-; SM60-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
-; SM60-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
-; SM60-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
-; SM60-NEXT: mov.b64 %rd33, %rd8;
-; SM60-NEXT: @%p20 bra $L__BB286_7;
+; SM60-NEXT: setp.eq.b64 %p13, %rd29, 0;
+; SM60-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM60-NEXT: max.f64 %rd23, %rd29, %rd2;
+; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
+; SM60-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
+; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
+; SM60-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
+; SM60-NEXT: mov.b64 %rd29, %rd8;
+; SM60-NEXT: @%p16 bra $L__BB286_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end19
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12620,8 +12516,8 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM60-LABEL: fmaximum_acq_rel_v8f64_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<41>;
-; SM60-NEXT: .reg .b64 %rd<66>;
+; SM60-NEXT: .reg .pred %p<33>;
+; SM60-NEXT: .reg .b64 %rd<58>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
@@ -12630,147 +12526,131 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM60-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
; SM60-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
; SM60-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd58, [%rd17];
-; SM60-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd50, [%rd17];
; SM60-NEXT: $L__BB287_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd7, %rd58;
-; SM60-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
-; SM60-NEXT: max.f64 %rd18, %rd7, %rd5;
-; SM60-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
-; SM60-NEXT: setp.eq.b64 %p2, %rd7, 0;
-; SM60-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
-; SM60-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
-; SM60-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
-; SM60-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM60-NEXT: @%p5 bra $L__BB287_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd59, [%rd17+8];
-; SM60-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM60-NEXT: mov.b64 %rd7, %rd50;
+; SM60-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM60-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM60-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM60-NEXT: max.f64 %rd19, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
+; SM60-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
+; SM60-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM60-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
+; SM60-NEXT: @%p4 bra $L__BB287_1;
+; SM60-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NEXT: ld.volatile.global.b64 %rd51, [%rd17+8];
; SM60-NEXT: $L__BB287_3: // %atomicrmw.start9
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd8, %rd59;
+; SM60-NEXT: mov.b64 %rd8, %rd51;
+; SM60-NEXT: setp.eq.b64 %p5, %rd8, 0;
+; SM60-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
; SM60-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
; SM60-NEXT: max.f64 %rd23, %rd8, %rd6;
; SM60-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM60-NEXT: setp.eq.b64 %p7, %rd8, 0;
-; SM60-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
-; SM60-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
-; SM60-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
-; SM60-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM60-NEXT: @%p10 bra $L__BB287_3;
+; SM60-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
+; SM60-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM60-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
+; SM60-NEXT: @%p8 bra $L__BB287_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end8
-; SM60-NEXT: ld.volatile.global.b64 %rd60, [%rd17+16];
-; SM60-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd52, [%rd17+16];
; SM60-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd9, %rd60;
-; SM60-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
-; SM60-NEXT: max.f64 %rd28, %rd9, %rd3;
-; SM60-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
-; SM60-NEXT: setp.eq.b64 %p12, %rd9, 0;
-; SM60-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
-; SM60-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
-; SM60-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
-; SM60-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM60-NEXT: @%p15 bra $L__BB287_5;
+; SM60-NEXT: mov.b64 %rd9, %rd52;
+; SM60-NEXT: setp.eq.b64 %p9, %rd9, 0;
+; SM60-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM60-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM60-NEXT: max.f64 %rd27, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
+; SM60-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
+; SM60-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM60-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
+; SM60-NEXT: @%p12 bra $L__BB287_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end18
-; SM60-NEXT: ld.volatile.global.b64 %rd61, [%rd17+24];
-; SM60-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd53, [%rd17+24];
; SM60-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd10, %rd61;
-; SM60-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
-; SM60-NEXT: max.f64 %rd33, %rd10, %rd4;
-; SM60-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
-; SM60-NEXT: setp.eq.b64 %p17, %rd10, 0;
-; SM60-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
-; SM60-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
-; SM60-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
-; SM60-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM60-NEXT: @%p20 bra $L__BB287_7;
+; SM60-NEXT: mov.b64 %rd10, %rd53;
+; SM60-NEXT: setp.eq.b64 %p13, %rd10, 0;
+; SM60-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM60-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM60-NEXT: max.f64 %rd31, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
+; SM60-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
+; SM60-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM60-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
+; SM60-NEXT: @%p16 bra $L__BB287_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end23
-; SM60-NEXT: ld.volatile.global.b64 %rd62, [%rd17+32];
-; SM60-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd54, [%rd17+32];
; SM60-NEXT: $L__BB287_9: // %atomicrmw.start38
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd13, %rd62;
-; SM60-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
-; SM60-NEXT: max.f64 %rd38, %rd13, %rd1;
-; SM60-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
-; SM60-NEXT: setp.eq.b64 %p22, %rd13, 0;
-; SM60-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
-; SM60-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
-; SM60-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
-; SM60-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM60-NEXT: @%p25 bra $L__BB287_9;
+; SM60-NEXT: mov.b64 %rd13, %rd54;
+; SM60-NEXT: setp.eq.b64 %p17, %rd13, 0;
+; SM60-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM60-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM60-NEXT: max.f64 %rd35, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
+; SM60-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
+; SM60-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM60-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
+; SM60-NEXT: @%p20 bra $L__BB287_9;
; SM60-NEXT: // %bb.10: // %atomicrmw.end37
-; SM60-NEXT: ld.volatile.global.b64 %rd63, [%rd17+40];
-; SM60-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd55, [%rd17+40];
; SM60-NEXT: $L__BB287_11: // %atomicrmw.start43
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b64 %rd14, %rd63;
-; SM60-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
-; SM60-NEXT: max.f64 %rd43, %rd14, %rd2;
-; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM60-NEXT: setp.eq.b64 %p27, %rd14, 0;
-; SM60-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
-; SM60-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
-; SM60-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
-; SM60-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM60-NEXT: @%p30 bra $L__BB287_11;
+; SM60-NEXT: mov.b64 %rd14, %rd55;
+; SM60-NEXT: setp.eq.b64 %p21, %rd14, 0;
+; SM60-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM60-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM60-NEXT: max.f64 %rd39, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
+; SM60-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
+; SM60-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM60-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
+; SM60-NEXT: @%p24 bra $L__BB287_11;
; SM60-NEXT: // %bb.12: // %atomicrmw.end42
-; SM60-NEXT: ld.volatile.global.b64 %rd64, [%rd17+48];
-; SM60-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd56, [%rd17+48];
; SM60-NEXT: $L__BB287_13: // %atomicrmw.start53
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
-; SM60-NEXT: max.f64 %rd48, %rd64, %rd11;
-; SM60-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
-; SM60-NEXT: setp.eq.b64 %p32, %rd64, 0;
-; SM60-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
-; SM60-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
-; SM60-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
-; SM60-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
-; SM60-NEXT: mov.b64 %rd64, %rd15;
-; SM60-NEXT: @%p35 bra $L__BB287_13;
+; SM60-NEXT: setp.eq.b64 %p25, %rd56, 0;
+; SM60-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM60-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM60-NEXT: max.f64 %rd43, %rd56, %rd11;
+; SM60-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM60-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
+; SM60-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM60-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM60-NEXT: mov.b64 %rd56, %rd15;
+; SM60-NEXT: @%p28 bra $L__BB287_13;
; SM60-NEXT: // %bb.14: // %atomicrmw.end52
-; SM60-NEXT: ld.volatile.global.b64 %rd65, [%rd17+56];
-; SM60-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM60-NEXT: ld.volatile.global.b64 %rd57, [%rd17+56];
; SM60-NEXT: $L__BB287_15: // %atomicrmw.start58
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
-; SM60-NEXT: max.f64 %rd53, %rd65, %rd12;
-; SM60-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
-; SM60-NEXT: setp.eq.b64 %p37, %rd65, 0;
-; SM60-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
-; SM60-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
-; SM60-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
-; SM60-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
-; SM60-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
-; SM60-NEXT: mov.b64 %rd65, %rd16;
-; SM60-NEXT: @%p40 bra $L__BB287_15;
+; SM60-NEXT: setp.eq.b64 %p29, %rd57, 0;
+; SM60-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM60-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM60-NEXT: max.f64 %rd47, %rd57, %rd12;
+; SM60-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
+; SM60-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
+; SM60-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
+; SM60-NEXT: atom.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
+; SM60-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
+; SM60-NEXT: mov.b64 %rd57, %rd16;
+; SM60-NEXT: @%p32 bra $L__BB287_15;
; SM60-NEXT: // %bb.16: // %atomicrmw.end57
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
-; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
-; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
+; SM60-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM60-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -13603,8 +13483,8 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM60-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<7>;
; SM60-NEXT: .reg .b32 %r<15>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
@@ -13620,29 +13500,26 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: shl.b32 %r7, %r6, %r1;
; SM60-NEXT: not.b32 %r2, %r7;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
; SM60-NEXT: $L__BB304_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r8, %r14, %r1;
; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, -32768;
; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: mov.b16 %rs7, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
; SM60-NEXT: shl.b32 %r10, %r9, %r1;
; SM60-NEXT: and.b32 %r11, %r14, %r2;
; SM60-NEXT: or.b32 %r12, %r11, %r10;
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r14;
; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p6 bra $L__BB304_1;
+; SM60-NEXT: @%p5 bra $L__BB304_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r13, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -13655,8 +13532,8 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<12>;
-; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<13>;
; SM60-NEXT: .reg .b32 %r<5>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
@@ -13666,35 +13543,30 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
; SM60-NEXT: $L__BB305_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
-; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p11 bra $L__BB305_1;
+; SM60-NEXT: @%p9 bra $L__BB305_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13706,8 +13578,8 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM60-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<22>;
-; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<25>;
; SM60-NEXT: .reg .b32 %r<7>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
@@ -13719,55 +13591,46 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
-; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: $L__BB306_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
-; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
-; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
-; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
-; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r6;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: setp.lt.f16 %p10, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM60-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: setp.lt.f16 %p14, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM60-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r4, {%rs24, %rs20};
; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -13776,10 +13639,10 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB306_1;
+; SM60-NEXT: @%p17 bra $L__BB306_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -13791,8 +13654,8 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM60-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<43>;
-; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .pred %p<35>;
+; SM60-NEXT: .reg .b16 %rs<49>;
; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
@@ -13804,55 +13667,46 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM60-NEXT: setp.eq.b16 %p14, %rs17, -32768;
-; SM60-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: $L__BB307_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM60-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r5, {%rs12, %rs8};
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
-; SM60-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs19, -32768;
-; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM60-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs18, -32768;
-; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r10;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: setp.lt.f16 %p10, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM60-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: setp.lt.f16 %p14, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM60-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r6, {%rs24, %rs20};
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -13861,64 +13715,55 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB307_1;
+; SM60-NEXT: @%p17 bra $L__BB307_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: $L__BB307_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
-; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
-; SM60-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
-; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
-; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
-; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
-; SM60-NEXT: setp.eq.b16 %p24, %rs33, -32768;
-; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
-; SM60-NEXT: setp.eq.b16 %p25, %rs31, -32768;
-; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
-; SM60-NEXT: mov.b16 %rs38, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
-; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
-; SM60-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
-; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
-; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
-; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
-; SM60-NEXT: setp.eq.b16 %p29, %rs32, -32768;
-; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
-; SM60-NEXT: setp.eq.b16 %p30, %rs30, -32768;
-; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
-; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
-; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
-; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r11;
+; SM60-NEXT: setp.eq.b16 %p18, %rs28, -32768;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM60-NEXT: setp.lt.f16 %p19, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM60-NEXT: setp.nan.f16 %p20, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs31, 0x7E00, %rs30, %p20;
+; SM60-NEXT: setp.eq.f16 %p21, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM60-NEXT: setp.eq.b16 %p22, %rs27, -32768;
+; SM60-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM60-NEXT: setp.lt.f16 %p23, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM60-NEXT: setp.nan.f16 %p24, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p24;
+; SM60-NEXT: setp.eq.f16 %p25, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM60-NEXT: mov.b32 %r7, {%rs36, %rs32};
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
-; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
-; SM60-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
-; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
-; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
-; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
-; SM60-NEXT: setp.eq.b16 %p34, %rs48, -32768;
-; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
-; SM60-NEXT: setp.eq.b16 %p35, %rs46, -32768;
-; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
-; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
-; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
-; SM60-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
-; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
-; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
-; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
-; SM60-NEXT: setp.eq.b16 %p39, %rs47, -32768;
-; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
-; SM60-NEXT: setp.eq.b16 %p40, %rs45, -32768;
-; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
-; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
-; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
-; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM60-NEXT: mov.b32 {%rs39, %rs40}, %r12;
+; SM60-NEXT: setp.eq.b16 %p26, %rs40, -32768;
+; SM60-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM60-NEXT: setp.lt.f16 %p27, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs43, 0x7E00, %rs42, %p28;
+; SM60-NEXT: setp.eq.f16 %p29, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs39, -32768;
+; SM60-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM60-NEXT: setp.lt.f16 %p31, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM60-NEXT: setp.nan.f16 %p32, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs47, 0x7E00, %rs46, %p32;
+; SM60-NEXT: setp.eq.f16 %p33, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM60-NEXT: mov.b32 %r8, {%rs48, %rs44};
; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
@@ -13927,10 +13772,10 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB307_3;
+; SM60-NEXT: @%p34 bra $L__BB307_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -13942,8 +13787,8 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM60-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<9>;
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<7>;
; SM60-NEXT: .reg .b32 %r<15>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
@@ -13959,29 +13804,26 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM60-NEXT: shl.b32 %r7, %r6, %r1;
; SM60-NEXT: not.b32 %r2, %r7;
; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
; SM60-NEXT: $L__BB308_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r8, %r14, %r1;
; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, 0;
; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: mov.b16 %rs7, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
; SM60-NEXT: shl.b32 %r10, %r9, %r1;
; SM60-NEXT: and.b32 %r11, %r14, %r2;
; SM60-NEXT: or.b32 %r12, %r11, %r10;
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r14;
; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p6 bra $L__BB308_1;
+; SM60-NEXT: @%p5 bra $L__BB308_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r13, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -13994,8 +13836,8 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<12>;
-; SM60-NEXT: .reg .b16 %rs<16>;
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<13>;
; SM60-NEXT: .reg .b32 %r<5>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
@@ -14005,35 +13847,30 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
; SM60-NEXT: $L__BB309_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r4;
-; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r4;
; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p11 bra $L__BB309_1;
+; SM60-NEXT: @%p9 bra $L__BB309_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -14045,8 +13882,8 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM60-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<22>;
-; SM60-NEXT: .reg .b16 %rs<30>;
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<25>;
; SM60-NEXT: .reg .b32 %r<7>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
@@ -14058,55 +13895,46 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
-; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: $L__BB310_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r5;
-; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM60-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r6;
-; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
-; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
-; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM60-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r6;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: setp.gt.f16 %p10, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM60-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: setp.gt.f16 %p14, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM60-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r4, {%rs24, %rs20};
; SM60-NEXT: cvt.u64.u32 %rd4, %r4;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -14115,10 +13943,10 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB310_1;
+; SM60-NEXT: @%p17 bra $L__BB310_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -14130,8 +13958,8 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM60-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<43>;
-; SM60-NEXT: .reg .b16 %rs<59>;
+; SM60-NEXT: .reg .pred %p<35>;
+; SM60-NEXT: .reg .b16 %rs<49>;
; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
@@ -14143,55 +13971,46 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM60-NEXT: setp.eq.b16 %p14, %rs17, 0;
-; SM60-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: $L__BB311_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM60-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM60-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: mov.b16 %rs9, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM60-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM60-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM60-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM60-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM60-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM60-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM60-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM60-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM60-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM60-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM60-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM60-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r5, {%rs12, %rs8};
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: mov.b32 {%rs18, %rs19}, %r10;
-; SM60-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM60-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM60-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs19, 0;
-; SM60-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM60-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM60-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM60-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM60-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM60-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM60-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs18, 0;
-; SM60-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM60-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM60-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM60-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r10;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: setp.gt.f16 %p10, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM60-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: setp.gt.f16 %p14, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM60-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r6, {%rs24, %rs20};
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -14200,64 +14019,55 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB311_1;
+; SM60-NEXT: @%p17 bra $L__BB311_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd12;
; SM60-NEXT: $L__BB311_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs30, %rs31}, %r3;
-; SM60-NEXT: mov.b32 {%rs32, %rs33}, %r11;
-; SM60-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
-; SM60-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
-; SM60-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
-; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
-; SM60-NEXT: setp.eq.b16 %p24, %rs33, 0;
-; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
-; SM60-NEXT: setp.eq.b16 %p25, %rs31, 0;
-; SM60-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
-; SM60-NEXT: mov.b16 %rs38, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
-; SM60-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
-; SM60-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
-; SM60-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
-; SM60-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
-; SM60-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
-; SM60-NEXT: setp.eq.b16 %p29, %rs32, 0;
-; SM60-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
-; SM60-NEXT: setp.eq.b16 %p30, %rs30, 0;
-; SM60-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
-; SM60-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
-; SM60-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
-; SM60-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r11;
+; SM60-NEXT: setp.eq.b16 %p18, %rs28, 0;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM60-NEXT: setp.gt.f16 %p19, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM60-NEXT: setp.nan.f16 %p20, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs31, 0x7E00, %rs30, %p20;
+; SM60-NEXT: setp.eq.f16 %p21, %rs28, %rs26;
+; SM60-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM60-NEXT: setp.eq.b16 %p22, %rs27, 0;
+; SM60-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM60-NEXT: setp.gt.f16 %p23, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM60-NEXT: setp.nan.f16 %p24, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p24;
+; SM60-NEXT: setp.eq.f16 %p25, %rs27, %rs25;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM60-NEXT: mov.b32 %r7, {%rs36, %rs32};
; SM60-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r4;
-; SM60-NEXT: mov.b32 {%rs47, %rs48}, %r12;
-; SM60-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
-; SM60-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
-; SM60-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
-; SM60-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
-; SM60-NEXT: setp.eq.b16 %p34, %rs48, 0;
-; SM60-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
-; SM60-NEXT: setp.eq.b16 %p35, %rs46, 0;
-; SM60-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
-; SM60-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
-; SM60-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
-; SM60-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
-; SM60-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
-; SM60-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
-; SM60-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
-; SM60-NEXT: setp.eq.b16 %p39, %rs47, 0;
-; SM60-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
-; SM60-NEXT: setp.eq.b16 %p40, %rs45, 0;
-; SM60-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
-; SM60-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
-; SM60-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
-; SM60-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM60-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM60-NEXT: mov.b32 {%rs39, %rs40}, %r12;
+; SM60-NEXT: setp.eq.b16 %p26, %rs40, 0;
+; SM60-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM60-NEXT: setp.gt.f16 %p27, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM60-NEXT: setp.nan.f16 %p28, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs43, 0x7E00, %rs42, %p28;
+; SM60-NEXT: setp.eq.f16 %p29, %rs40, %rs38;
+; SM60-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs39, 0;
+; SM60-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM60-NEXT: setp.gt.f16 %p31, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM60-NEXT: setp.nan.f16 %p32, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs47, 0x7E00, %rs46, %p32;
+; SM60-NEXT: setp.eq.f16 %p33, %rs39, %rs37;
+; SM60-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM60-NEXT: mov.b32 %r8, {%rs48, %rs44};
; SM60-NEXT: cvt.u64.u32 %rd14, %r8;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
@@ -14266,10 +14076,10 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM60-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB311_3;
+; SM60-NEXT: @%p34 bra $L__BB311_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -15669,9 +15479,9 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM60-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<8>;
-; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<18>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -15685,38 +15495,34 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 %r6, 65535;
; SM60-NEXT: shl.b32 %r7, %r6, %r1;
; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
; SM60-NEXT: $L__BB328_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, -32768;
; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM60-NEXT: shl.b32 %r15, %r14, %r1;
-; SM60-NEXT: and.b32 %r16, %r19, %r2;
-; SM60-NEXT: or.b32 %r17, %r16, %r15;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM60-NEXT: mov.b32 %r19, %r3;
-; SM60-NEXT: @%p6 bra $L__BB328_1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p5 bra $L__BB328_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -15725,57 +15531,49 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<12>;
-; SM60-NEXT: .reg .b16 %rs<15>;
-; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: ld.volatile.global.b32 %r12, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: $L__BB329_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: setp.lt.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r10, %r8;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
-; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
-; SM60-NEXT: mov.b32 %r16, %r1;
-; SM60-NEXT: @%p11 bra $L__BB329_1;
+; SM60-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r12;
+; SM60-NEXT: mov.b32 %r12, %r1;
+; SM60-NEXT: @%p9 bra $L__BB329_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -15787,9 +15585,9 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM60-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<22>;
-; SM60-NEXT: .reg .b16 %rs<29>;
-; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<23>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -15797,96 +15595,80 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs14;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
; SM60-NEXT: shl.b32 %r17, %r16, 16;
-; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
-; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
-; SM60-NEXT: shl.b32 %r23, %r22, 16;
-; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
; SM60-NEXT: $L__BB330_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.lt.f32 %p1, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM60-NEXT: setp.lt.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.lt.f32 %p6, %r10, %r8;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
-; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
-; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r22;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs16;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
+; SM60-NEXT: setp.lt.f32 %p10, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f32 %p11, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM60-NEXT: setp.eq.f32 %p12, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs15;
; SM60-NEXT: shl.b32 %r19, %r18, 16;
-; SM60-NEXT: setp.lt.f32 %p11, %r19, %r17;
-; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
-; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
-; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
-; SM60-NEXT: shl.b32 %r21, %r20, 16;
-; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
-; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
-; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: setp.lt.f32 %p16, %r25, %r23;
-; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
-; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
-; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
-; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
-; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: setp.lt.f32 %p14, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f32 %p15, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM60-NEXT: setp.eq.f32 %p16, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r20, {%rs24, %rs20};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r20;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB330_1;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB330_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -15895,9 +15677,9 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM60-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<43>;
-; SM60-NEXT: .reg .b16 %rs<57>;
-; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .pred %p<35>;
+; SM60-NEXT: .reg .b16 %rs<49>;
+; SM60-NEXT: .reg .b32 %r<45>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -15905,185 +15687,153 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
-; SM60-NEXT: shl.b32 %r19, %r18, 16;
-; SM60-NEXT: setp.eq.b16 %p14, %rs16, -32768;
-; SM60-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: $L__BB331_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r41;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.lt.f32 %p1, %r8, %r6;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: setp.lt.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM60-NEXT: setp.lt.f32 %p6, %r12, %r10;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
-; SM60-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
-; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: setp.eq.f32 %p8, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r13, {%rs12, %rs8};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r42;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.lt.f32 %p10, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f32 %p11, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM60-NEXT: setp.eq.f32 %p12, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs13;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
; SM60-NEXT: shl.b32 %r21, %r20, 16;
-; SM60-NEXT: setp.lt.f32 %p11, %r21, %r19;
-; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
-; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs18, -32768;
-; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
-; SM60-NEXT: shl.b32 %r23, %r22, 16;
-; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
-; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
-; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: setp.lt.f32 %p16, %r27, %r25;
-; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
-; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs17, -32768;
-; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
-; SM60-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
-; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
-; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: setp.lt.f32 %p14, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f32 %p15, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM60-NEXT: setp.eq.f32 %p16, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r22, {%rs24, %rs20};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r22;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r42;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB331_1;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB331_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd12;
; SM60-NEXT: $L__BB331_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
-; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
-; SM60-NEXT: shl.b32 %r32, %r31, 16;
-; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
-; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
-; SM60-NEXT: shl.b32 %r34, %r33, 16;
-; SM60-NEXT: setp.lt.f32 %p22, %r34, %r32;
-; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
-; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
-; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
-; SM60-NEXT: setp.eq.b16 %p24, %rs32, -32768;
-; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
-; SM60-NEXT: setp.eq.b16 %p25, %rs30, -32768;
-; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
-; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
-; SM60-NEXT: shl.b32 %r36, %r35, 16;
-; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
-; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
-; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
-; SM60-NEXT: shl.b32 %r38, %r37, 16;
-; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
-; SM60-NEXT: shl.b32 %r40, %r39, 16;
-; SM60-NEXT: setp.lt.f32 %p27, %r40, %r38;
-; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
-; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
-; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
-; SM60-NEXT: setp.eq.b16 %p29, %rs31, -32768;
-; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
-; SM60-NEXT: setp.eq.b16 %p30, %rs29, -32768;
-; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
-; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
-; SM60-NEXT: shl.b32 %r42, %r41, 16;
-; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
-; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
-; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
-; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
-; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
-; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
-; SM60-NEXT: shl.b32 %r45, %r44, 16;
-; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
-; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
-; SM60-NEXT: shl.b32 %r47, %r46, 16;
-; SM60-NEXT: setp.lt.f32 %p32, %r47, %r45;
-; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
-; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
-; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
-; SM60-NEXT: setp.eq.b16 %p34, %rs46, -32768;
-; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
-; SM60-NEXT: setp.eq.b16 %p35, %rs44, -32768;
-; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
-; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
-; SM60-NEXT: shl.b32 %r49, %r48, 16;
-; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
-; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
-; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
-; SM60-NEXT: shl.b32 %r51, %r50, 16;
-; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
-; SM60-NEXT: shl.b32 %r53, %r52, 16;
-; SM60-NEXT: setp.lt.f32 %p37, %r53, %r51;
-; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
-; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
-; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
-; SM60-NEXT: setp.eq.b16 %p39, %rs45, -32768;
-; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
-; SM60-NEXT: setp.eq.b16 %p40, %rs43, -32768;
-; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
-; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
-; SM60-NEXT: shl.b32 %r55, %r54, 16;
-; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
-; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
-; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
-; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r43;
+; SM60-NEXT: setp.eq.b16 %p18, %rs28, -32768;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs26;
+; SM60-NEXT: shl.b32 %r24, %r23, 16;
+; SM60-NEXT: cvt.u32.u16 %r25, %rs28;
+; SM60-NEXT: shl.b32 %r26, %r25, 16;
+; SM60-NEXT: setp.lt.f32 %p19, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM60-NEXT: setp.nan.f32 %p20, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs31, 0x7FC0, %rs30, %p20;
+; SM60-NEXT: setp.eq.f32 %p21, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM60-NEXT: setp.eq.b16 %p22, %rs27, -32768;
+; SM60-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs25;
+; SM60-NEXT: shl.b32 %r28, %r27, 16;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs27;
+; SM60-NEXT: shl.b32 %r30, %r29, 16;
+; SM60-NEXT: setp.lt.f32 %p23, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM60-NEXT: setp.nan.f32 %p24, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs35, 0x7FC0, %rs34, %p24;
+; SM60-NEXT: setp.eq.f32 %p25, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM60-NEXT: mov.b32 %r31, {%rs36, %rs32};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r31;
+; SM60-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM60-NEXT: mov.b32 {%rs39, %rs40}, %r44;
+; SM60-NEXT: setp.eq.b16 %p26, %rs40, -32768;
+; SM60-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM60-NEXT: cvt.u32.u16 %r32, %rs38;
+; SM60-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs40;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: setp.lt.f32 %p27, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs43, 0x7FC0, %rs42, %p28;
+; SM60-NEXT: setp.eq.f32 %p29, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs39, -32768;
+; SM60-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs37;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs39;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: setp.lt.f32 %p31, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM60-NEXT: setp.nan.f32 %p32, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs47, 0x7FC0, %rs46, %p32;
+; SM60-NEXT: setp.eq.f32 %p33, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM60-NEXT: mov.b32 %r40, {%rs48, %rs44};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r40;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r44;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB331_3;
+; SM60-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM60-NEXT: @%p34 bra $L__BB331_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -16092,9 +15842,9 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM60-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<8>;
-; SM60-NEXT: .reg .b32 %r<20>;
+; SM60-NEXT: .reg .pred %p<6>;
+; SM60-NEXT: .reg .b16 %rs<7>;
+; SM60-NEXT: .reg .b32 %r<18>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -16108,38 +15858,34 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 %r6, 65535;
; SM60-NEXT: shl.b32 %r7, %r6, %r1;
; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
; SM60-NEXT: $L__BB332_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NEXT: shr.u32 %r8, %r17, %r1;
; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM60-NEXT: setp.eq.b16 %p1, %rs2, 0;
; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM60-NEXT: shl.b32 %r15, %r14, %r1;
-; SM60-NEXT: and.b32 %r16, %r19, %r2;
-; SM60-NEXT: or.b32 %r17, %r16, %r15;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM60-NEXT: mov.b32 %r19, %r3;
-; SM60-NEXT: @%p6 bra $L__BB332_1;
+; SM60-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NEXT: mov.b32 %r17, %r3;
+; SM60-NEXT: @%p5 bra $L__BB332_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NEXT: shr.u32 %r16, %r3, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NEXT: st.param.b16 [func_retval0], %r16;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -16148,57 +15894,49 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM60-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<12>;
-; SM60-NEXT: .reg .b16 %rs<15>;
-; SM60-NEXT: .reg .b32 %r<17>;
+; SM60-NEXT: .reg .pred %p<10>;
+; SM60-NEXT: .reg .b16 %rs<13>;
+; SM60-NEXT: .reg .b32 %r<13>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
+; SM60-NEXT: ld.volatile.global.b32 %r12, [%rd1];
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: $L__BB333_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: setp.gt.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r10, %r8;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
-; SM60-NEXT: setp.ne.b32 %p11, %r1, %r16;
-; SM60-NEXT: mov.b32 %r16, %r1;
-; SM60-NEXT: @%p11 bra $L__BB333_1;
+; SM60-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
+; SM60-NEXT: setp.ne.b32 %p9, %r1, %r12;
+; SM60-NEXT: mov.b32 %r12, %r1;
+; SM60-NEXT: @%p9 bra $L__BB333_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -16210,9 +15948,9 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM60-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<22>;
-; SM60-NEXT: .reg .b16 %rs<29>;
-; SM60-NEXT: .reg .b32 %r<31>;
+; SM60-NEXT: .reg .pred %p<18>;
+; SM60-NEXT: .reg .b16 %rs<25>;
+; SM60-NEXT: .reg .b32 %r<23>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -16220,96 +15958,80 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r12, %rs14;
+; SM60-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
; SM60-NEXT: shl.b32 %r17, %r16, 16;
-; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
-; SM60-NEXT: cvt.u32.u16 %r22, %rs15;
-; SM60-NEXT: shl.b32 %r23, %r22, 16;
-; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
; SM60-NEXT: $L__BB334_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.gt.f32 %p1, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM60-NEXT: setp.gt.f32 %p2, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NEXT: shl.b32 %r10, %r9, 16;
+; SM60-NEXT: setp.gt.f32 %p6, %r10, %r8;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM60-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM60-NEXT: cvt.u64.u32 %rd3, %r15;
-; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r30;
-; SM60-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM60-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r22;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs16;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
+; SM60-NEXT: setp.gt.f32 %p10, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f32 %p11, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM60-NEXT: setp.eq.f32 %p12, %r15, %r13;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs15;
; SM60-NEXT: shl.b32 %r19, %r18, 16;
-; SM60-NEXT: setp.gt.f32 %p11, %r19, %r17;
-; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM60-NEXT: setp.nan.f32 %p12, %r19, %r17;
-; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
-; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
-; SM60-NEXT: shl.b32 %r21, %r20, 16;
-; SM60-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
-; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM60-NEXT: cvt.u32.u16 %r24, %rs17;
-; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: setp.gt.f32 %p16, %r25, %r23;
-; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM60-NEXT: setp.nan.f32 %p17, %r25, %r23;
-; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
-; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs25;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
-; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM60-NEXT: mov.b32 %r28, {%rs28, %rs23};
-; SM60-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM60-NEXT: setp.gt.f32 %p14, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f32 %p15, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM60-NEXT: setp.eq.f32 %p16, %r19, %r17;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r20, {%rs24, %rs20};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r20;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r29;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r22;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB334_1;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB334_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -16318,9 +16040,9 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM60-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM60: {
-; SM60-NEXT: .reg .pred %p<43>;
-; SM60-NEXT: .reg .b16 %rs<57>;
-; SM60-NEXT: .reg .b32 %r<61>;
+; SM60-NEXT: .reg .pred %p<35>;
+; SM60-NEXT: .reg .b16 %rs<49>;
+; SM60-NEXT: .reg .b32 %r<45>;
; SM60-NEXT: .reg .b64 %rd<22>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
@@ -16328,185 +16050,153 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
; SM60-NEXT: shl.b32 %r6, %r5, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM60-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r18, %rs16;
-; SM60-NEXT: shl.b32 %r19, %r18, 16;
-; SM60-NEXT: setp.eq.b16 %p14, %rs16, 0;
-; SM60-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: $L__BB335_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r41;
+; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
-; SM60-NEXT: setp.gt.f32 %p1, %r8, %r6;
-; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r8, %r6;
-; SM60-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM60-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM60-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NEXT: setp.gt.f32 %p2, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM60-NEXT: setp.nan.f32 %p3, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM60-NEXT: setp.eq.f32 %p4, %r8, %r6;
+; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r9, %rs1;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
-; SM60-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM60-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM60-NEXT: cvt.u32.u16 %r11, %rs3;
; SM60-NEXT: shl.b32 %r12, %r11, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs3;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
-; SM60-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM60-NEXT: setp.gt.f32 %p6, %r12, %r10;
; SM60-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM60-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM60-NEXT: setp.nan.f32 %p7, %r12, %r10;
; SM60-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM60-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM60-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM60-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM60-NEXT: cvt.u32.u16 %r15, %rs11;
-; SM60-NEXT: shl.b32 %r16, %r15, 16;
-; SM60-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
-; SM60-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM60-NEXT: mov.b32 %r17, {%rs14, %rs9};
-; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM60-NEXT: mov.b32 {%rs17, %rs18}, %r58;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM60-NEXT: setp.eq.f32 %p8, %r12, %r10;
+; SM60-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM60-NEXT: mov.b32 %r13, {%rs12, %rs8};
+; SM60-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r42;
+; SM60-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
+; SM60-NEXT: setp.gt.f32 %p10, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM60-NEXT: setp.nan.f32 %p11, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM60-NEXT: setp.eq.f32 %p12, %r17, %r15;
+; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM60-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r18, %rs13;
+; SM60-NEXT: shl.b32 %r19, %r18, 16;
+; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
; SM60-NEXT: shl.b32 %r21, %r20, 16;
-; SM60-NEXT: setp.gt.f32 %p11, %r21, %r19;
-; SM60-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM60-NEXT: setp.nan.f32 %p12, %r21, %r19;
-; SM60-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM60-NEXT: setp.eq.b16 %p13, %rs18, 0;
-; SM60-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM60-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM60-NEXT: cvt.u32.u16 %r22, %rs20;
-; SM60-NEXT: shl.b32 %r23, %r22, 16;
-; SM60-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
-; SM60-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM60-NEXT: cvt.u32.u16 %r24, %rs15;
-; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs17;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
-; SM60-NEXT: setp.gt.f32 %p16, %r27, %r25;
-; SM60-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM60-NEXT: setp.nan.f32 %p17, %r27, %r25;
-; SM60-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM60-NEXT: setp.eq.b16 %p18, %rs17, 0;
-; SM60-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM60-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM60-NEXT: cvt.u32.u16 %r28, %rs25;
-; SM60-NEXT: shl.b32 %r29, %r28, 16;
-; SM60-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
-; SM60-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM60-NEXT: mov.b32 %r30, {%rs28, %rs23};
-; SM60-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM60-NEXT: setp.gt.f32 %p14, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM60-NEXT: setp.nan.f32 %p15, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM60-NEXT: setp.eq.f32 %p16, %r21, %r19;
+; SM60-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM60-NEXT: mov.b32 %r22, {%rs24, %rs20};
+; SM60-NEXT: cvt.u64.u32 %rd4, %r22;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r57;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM60-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM60-NEXT: cvt.u64.u32 %rd8, %r42;
; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM60-NEXT: @%p21 bra $L__BB335_1;
+; SM60-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM60-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM60-NEXT: @%p17 bra $L__BB335_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd12, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd12;
; SM60-NEXT: $L__BB335_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: mov.b32 {%rs29, %rs30}, %r3;
-; SM60-NEXT: cvt.u32.u16 %r31, %rs30;
-; SM60-NEXT: shl.b32 %r32, %r31, 16;
-; SM60-NEXT: mov.b32 {%rs31, %rs32}, %r59;
-; SM60-NEXT: cvt.u32.u16 %r33, %rs32;
-; SM60-NEXT: shl.b32 %r34, %r33, 16;
-; SM60-NEXT: setp.gt.f32 %p22, %r34, %r32;
-; SM60-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
-; SM60-NEXT: setp.nan.f32 %p23, %r34, %r32;
-; SM60-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
-; SM60-NEXT: setp.eq.b16 %p24, %rs32, 0;
-; SM60-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
-; SM60-NEXT: setp.eq.b16 %p25, %rs30, 0;
-; SM60-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
-; SM60-NEXT: cvt.u32.u16 %r35, %rs34;
-; SM60-NEXT: shl.b32 %r36, %r35, 16;
-; SM60-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
-; SM60-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
-; SM60-NEXT: cvt.u32.u16 %r37, %rs29;
-; SM60-NEXT: shl.b32 %r38, %r37, 16;
-; SM60-NEXT: cvt.u32.u16 %r39, %rs31;
-; SM60-NEXT: shl.b32 %r40, %r39, 16;
-; SM60-NEXT: setp.gt.f32 %p27, %r40, %r38;
-; SM60-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
-; SM60-NEXT: setp.nan.f32 %p28, %r40, %r38;
-; SM60-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
-; SM60-NEXT: setp.eq.b16 %p29, %rs31, 0;
-; SM60-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
-; SM60-NEXT: setp.eq.b16 %p30, %rs29, 0;
-; SM60-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
-; SM60-NEXT: cvt.u32.u16 %r41, %rs39;
-; SM60-NEXT: shl.b32 %r42, %r41, 16;
-; SM60-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
-; SM60-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
-; SM60-NEXT: mov.b32 %r43, {%rs42, %rs37};
-; SM60-NEXT: cvt.u64.u32 %rd13, %r43;
-; SM60-NEXT: mov.b32 {%rs43, %rs44}, %r4;
-; SM60-NEXT: cvt.u32.u16 %r44, %rs44;
-; SM60-NEXT: shl.b32 %r45, %r44, 16;
-; SM60-NEXT: mov.b32 {%rs45, %rs46}, %r60;
-; SM60-NEXT: cvt.u32.u16 %r46, %rs46;
-; SM60-NEXT: shl.b32 %r47, %r46, 16;
-; SM60-NEXT: setp.gt.f32 %p32, %r47, %r45;
-; SM60-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
-; SM60-NEXT: setp.nan.f32 %p33, %r47, %r45;
-; SM60-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
-; SM60-NEXT: setp.eq.b16 %p34, %rs46, 0;
-; SM60-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
-; SM60-NEXT: setp.eq.b16 %p35, %rs44, 0;
-; SM60-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
-; SM60-NEXT: cvt.u32.u16 %r48, %rs48;
-; SM60-NEXT: shl.b32 %r49, %r48, 16;
-; SM60-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
-; SM60-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
-; SM60-NEXT: cvt.u32.u16 %r50, %rs43;
-; SM60-NEXT: shl.b32 %r51, %r50, 16;
-; SM60-NEXT: cvt.u32.u16 %r52, %rs45;
-; SM60-NEXT: shl.b32 %r53, %r52, 16;
-; SM60-NEXT: setp.gt.f32 %p37, %r53, %r51;
-; SM60-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
-; SM60-NEXT: setp.nan.f32 %p38, %r53, %r51;
-; SM60-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
-; SM60-NEXT: setp.eq.b16 %p39, %rs45, 0;
-; SM60-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
-; SM60-NEXT: setp.eq.b16 %p40, %rs43, 0;
-; SM60-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
-; SM60-NEXT: cvt.u32.u16 %r54, %rs53;
-; SM60-NEXT: shl.b32 %r55, %r54, 16;
-; SM60-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
-; SM60-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
-; SM60-NEXT: mov.b32 %r56, {%rs56, %rs51};
-; SM60-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM60-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM60-NEXT: mov.b32 {%rs27, %rs28}, %r43;
+; SM60-NEXT: setp.eq.b16 %p18, %rs28, 0;
+; SM60-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM60-NEXT: cvt.u32.u16 %r23, %rs26;
+; SM60-NEXT: shl.b32 %r24, %r23, 16;
+; SM60-NEXT: cvt.u32.u16 %r25, %rs28;
+; SM60-NEXT: shl.b32 %r26, %r25, 16;
+; SM60-NEXT: setp.gt.f32 %p19, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM60-NEXT: setp.nan.f32 %p20, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs31, 0x7FC0, %rs30, %p20;
+; SM60-NEXT: setp.eq.f32 %p21, %r26, %r24;
+; SM60-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM60-NEXT: setp.eq.b16 %p22, %rs27, 0;
+; SM60-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM60-NEXT: cvt.u32.u16 %r27, %rs25;
+; SM60-NEXT: shl.b32 %r28, %r27, 16;
+; SM60-NEXT: cvt.u32.u16 %r29, %rs27;
+; SM60-NEXT: shl.b32 %r30, %r29, 16;
+; SM60-NEXT: setp.gt.f32 %p23, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM60-NEXT: setp.nan.f32 %p24, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs35, 0x7FC0, %rs34, %p24;
+; SM60-NEXT: setp.eq.f32 %p25, %r30, %r28;
+; SM60-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM60-NEXT: mov.b32 %r31, {%rs36, %rs32};
+; SM60-NEXT: cvt.u64.u32 %rd13, %r31;
+; SM60-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM60-NEXT: mov.b32 {%rs39, %rs40}, %r44;
+; SM60-NEXT: setp.eq.b16 %p26, %rs40, 0;
+; SM60-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM60-NEXT: cvt.u32.u16 %r32, %rs38;
+; SM60-NEXT: shl.b32 %r33, %r32, 16;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs40;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
+; SM60-NEXT: setp.gt.f32 %p27, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM60-NEXT: setp.nan.f32 %p28, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs43, 0x7FC0, %rs42, %p28;
+; SM60-NEXT: setp.eq.f32 %p29, %r35, %r33;
+; SM60-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM60-NEXT: setp.eq.b16 %p30, %rs39, 0;
+; SM60-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM60-NEXT: cvt.u32.u16 %r36, %rs37;
+; SM60-NEXT: shl.b32 %r37, %r36, 16;
+; SM60-NEXT: cvt.u32.u16 %r38, %rs39;
+; SM60-NEXT: shl.b32 %r39, %r38, 16;
+; SM60-NEXT: setp.gt.f32 %p31, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM60-NEXT: setp.nan.f32 %p32, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs47, 0x7FC0, %rs46, %p32;
+; SM60-NEXT: setp.eq.f32 %p33, %r39, %r37;
+; SM60-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM60-NEXT: mov.b32 %r40, {%rs48, %rs44};
+; SM60-NEXT: cvt.u64.u32 %rd14, %r40;
; SM60-NEXT: shl.b64 %rd15, %rd14, 32;
; SM60-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM60-NEXT: cvt.u64.u32 %rd17, %r59;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM60-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM60-NEXT: cvt.u64.u32 %rd18, %r44;
; SM60-NEXT: shl.b64 %rd19, %rd18, 32;
; SM60-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM60-NEXT: atom.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM60-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
-; SM60-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM60-NEXT: @%p42 bra $L__BB335_3;
+; SM60-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
+; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM60-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM60-NEXT: @%p34 bra $L__BB335_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM60-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index 4a4589fe80f56..a2a37ec0bb265 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -9919,88 +9919,88 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
-define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
-; SM70-LABEL: fadd_acq_rel_v1f32_global_cta(
+define <1 x float> @fadd_acq_rel_v1f32_shared_cta(ptr addrspace(3) %addr, <1 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v1f32_shared_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<3>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_shared_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_shared_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r2, [%rd1], %r1;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
}
-define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
-; SM70-LABEL: fadd_acq_rel_v2f32_global_cta(
+define <2 x float> @fadd_acq_rel_v2f32_shared_cta(ptr addrspace(3) %addr, <2 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v2f32_shared_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_shared_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_shared_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r3, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r4, [%rd1+4], %r2;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
}
-define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
-; SM70-LABEL: fadd_acq_rel_v4f32_global_cta(
+define <4 x float> @fadd_acq_rel_v4f32_shared_cta(ptr addrspace(3) %addr, <4 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v4f32_shared_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_shared_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_shared_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r5, [%rd1], %r1;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r6, [%rd1+4], %r2;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r7, [%rd1+8], %r3;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r8, [%rd1+12], %r4;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
}
-define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
-; SM70-LABEL: fadd_acq_rel_v8f32_global_cta(
+define <8 x float> @fadd_acq_rel_v8f32_shared_cta(ptr addrspace(3) %addr, <8 x float> %val) {
+; SM70-LABEL: fadd_acq_rel_v8f32_shared_cta(
; SM70: {
; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_shared_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_shared_cta_param_1+16];
+; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_shared_cta_param_1];
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r9, [%rd1], %r5;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r10, [%rd1+4], %r6;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r11, [%rd1+8], %r7;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r12, [%rd1+12], %r8;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r13, [%rd1+16], %r1;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r14, [%rd1+20], %r2;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r15, [%rd1+24], %r3;
+; SM70-NEXT: atom.relaxed.cta.shared.add.f32 %r16, [%rd1+28], %r4;
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
}
@@ -10697,30 +10697,28 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<8>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
-; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NEXT: $L__BB256_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
-; SM70-NEXT: min.f32 %r3, %r8, %r1;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
-; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
-; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
-; SM70-NEXT: mov.b32 %r8, %r2;
-; SM70-NEXT: @%p5 bra $L__BB256_1;
+; SM70-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM70-NEXT: selp.f32 %r3, %r7, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r7, %r1;
+; SM70-NEXT: min.f32 %r4, %r7, %r1;
+; SM70-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
+; SM70-NEXT: setp.ne.b32 %p4, %r2, %r7;
+; SM70-NEXT: mov.b32 %r7, %r2;
+; SM70-NEXT: @%p4 bra $L__BB256_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
@@ -10732,8 +10730,8 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<10>;
-; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .pred %p<8>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -10741,44 +10739,40 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd2;
; SM70-NEXT: $L__BB257_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
-; SM70-NEXT: min.f32 %r3, %r13, %r1;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r13, -2147483648;
-; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
-; SM70-NEXT: min.f32 %r8, %r14, %r2;
+; SM70-NEXT: setp.eq.b32 %p1, %r11, -2147483648;
+; SM70-NEXT: selp.f32 %r3, %r11, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r11, %r1;
+; SM70-NEXT: min.f32 %r4, %r11, %r1;
+; SM70-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r11, %r1;
+; SM70-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: setp.eq.b32 %p4, %r12, -2147483648;
+; SM70-NEXT: selp.f32 %r7, %r12, %r2, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r12, %r2;
+; SM70-NEXT: min.f32 %r8, %r12, %r2;
; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r14, -2147483648;
-; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
-; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
-; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: setp.eq.f32 %p6, %r12, %r2;
+; SM70-NEXT: selp.f32 %r10, %r7, %r9, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB257_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB257_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10787,8 +10781,8 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<19>;
-; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .pred %p<15>;
+; SM70-NEXT: .reg .b32 %r<25>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -10796,81 +10790,73 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r1, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p7, %r2, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
; SM70-NEXT: $L__BB258_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
-; SM70-NEXT: min.f32 %r5, %r25, %r1;
-; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r25, -2147483648;
-; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
-; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
-; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
-; SM70-NEXT: min.f32 %r10, %r26, %r2;
+; SM70-NEXT: setp.eq.b32 %p1, %r21, -2147483648;
+; SM70-NEXT: selp.f32 %r5, %r21, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r21, %r1;
+; SM70-NEXT: min.f32 %r6, %r21, %r1;
+; SM70-NEXT: selp.f32 %r7, 0f7FC00000, %r6, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r21, %r1;
+; SM70-NEXT: selp.f32 %r8, %r5, %r7, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM70-NEXT: setp.eq.b32 %p4, %r22, -2147483648;
+; SM70-NEXT: selp.f32 %r9, %r22, %r2, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r22, %r2;
+; SM70-NEXT: min.f32 %r10, %r22, %r2;
; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r26, -2147483648;
-; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
-; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
-; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: setp.eq.f32 %p6, %r22, %r2;
+; SM70-NEXT: selp.f32 %r12, %r9, %r11, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB258_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB258_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM70-NEXT: setp.eq.b32 %p12, %r3, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p16, %r4, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd12;
; SM70-NEXT: $L__BB258_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
-; SM70-NEXT: min.f32 %r15, %r27, %r3;
-; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
-; SM70-NEXT: setp.eq.b32 %p11, %r27, -2147483648;
-; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
-; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
-; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
-; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
-; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
-; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
-; SM70-NEXT: min.f32 %r20, %r28, %r4;
-; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
-; SM70-NEXT: setp.eq.b32 %p15, %r28, -2147483648;
-; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
-; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
-; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
-; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
-; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: setp.eq.b32 %p8, %r23, -2147483648;
+; SM70-NEXT: selp.f32 %r13, %r23, %r3, %p8;
+; SM70-NEXT: setp.nan.f32 %p9, %r23, %r3;
+; SM70-NEXT: min.f32 %r14, %r23, %r3;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p9;
+; SM70-NEXT: setp.eq.f32 %p10, %r23, %r3;
+; SM70-NEXT: selp.f32 %r16, %r13, %r15, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM70-NEXT: setp.eq.b32 %p11, %r24, -2147483648;
+; SM70-NEXT: selp.f32 %r17, %r24, %r4, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r24, %r4;
+; SM70-NEXT: min.f32 %r18, %r24, %r4;
+; SM70-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r24, %r4;
+; SM70-NEXT: selp.f32 %r20, %r17, %r19, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r20;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r24;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB258_3;
+; SM70-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM70-NEXT: @%p14 bra $L__BB258_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10879,8 +10865,8 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<37>;
-; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .pred %p<29>;
+; SM70-NEXT: .reg .b32 %r<49>;
; SM70-NEXT: .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -10889,156 +10875,140 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r5, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p7, %r6, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: $L__BB259_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
-; SM70-NEXT: min.f32 %r9, %r49, %r5;
-; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r49, -2147483648;
-; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
-; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
-; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
-; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
-; SM70-NEXT: min.f32 %r14, %r50, %r6;
+; SM70-NEXT: setp.eq.b32 %p1, %r41, -2147483648;
+; SM70-NEXT: selp.f32 %r9, %r41, %r5, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r41, %r5;
+; SM70-NEXT: min.f32 %r10, %r41, %r5;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r41, %r5;
+; SM70-NEXT: selp.f32 %r12, %r9, %r11, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r12;
+; SM70-NEXT: setp.eq.b32 %p4, %r42, -2147483648;
+; SM70-NEXT: selp.f32 %r13, %r42, %r6, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r42, %r6;
+; SM70-NEXT: min.f32 %r14, %r42, %r6;
; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r50, -2147483648;
-; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
-; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
-; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: setp.eq.f32 %p6, %r42, %r6;
+; SM70-NEXT: selp.f32 %r16, %r13, %r15, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r42;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB259_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB259_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
-; SM70-NEXT: setp.eq.b32 %p12, %r7, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p16, %r8, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd12;
; SM70-NEXT: $L__BB259_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
-; SM70-NEXT: min.f32 %r19, %r51, %r7;
-; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
-; SM70-NEXT: setp.eq.b32 %p11, %r51, -2147483648;
-; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
-; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
-; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
-; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
-; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
-; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
-; SM70-NEXT: min.f32 %r24, %r52, %r8;
-; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
-; SM70-NEXT: setp.eq.b32 %p15, %r52, -2147483648;
-; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
-; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
-; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
-; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
-; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: setp.eq.b32 %p8, %r43, -2147483648;
+; SM70-NEXT: selp.f32 %r17, %r43, %r7, %p8;
+; SM70-NEXT: setp.nan.f32 %p9, %r43, %r7;
+; SM70-NEXT: min.f32 %r18, %r43, %r7;
+; SM70-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p9;
+; SM70-NEXT: setp.eq.f32 %p10, %r43, %r7;
+; SM70-NEXT: selp.f32 %r20, %r17, %r19, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r20;
+; SM70-NEXT: setp.eq.b32 %p11, %r44, -2147483648;
+; SM70-NEXT: selp.f32 %r21, %r44, %r8, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r44, %r8;
+; SM70-NEXT: min.f32 %r22, %r44, %r8;
+; SM70-NEXT: selp.f32 %r23, 0f7FC00000, %r22, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r44, %r8;
+; SM70-NEXT: selp.f32 %r24, %r21, %r23, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r44;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB259_3;
+; SM70-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM70-NEXT: @%p14 bra $L__BB259_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
-; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
-; SM70-NEXT: setp.eq.b32 %p21, %r1, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p25, %r2, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd22;
; SM70-NEXT: $L__BB259_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
-; SM70-NEXT: min.f32 %r29, %r53, %r1;
-; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
-; SM70-NEXT: setp.eq.b32 %p20, %r53, -2147483648;
-; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
-; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
-; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
-; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
-; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
-; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
-; SM70-NEXT: min.f32 %r34, %r54, %r2;
-; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
-; SM70-NEXT: setp.eq.b32 %p24, %r54, -2147483648;
-; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
-; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
-; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
-; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
-; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: setp.eq.b32 %p15, %r45, -2147483648;
+; SM70-NEXT: selp.f32 %r25, %r45, %r1, %p15;
+; SM70-NEXT: setp.nan.f32 %p16, %r45, %r1;
+; SM70-NEXT: min.f32 %r26, %r45, %r1;
+; SM70-NEXT: selp.f32 %r27, 0f7FC00000, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r45, %r1;
+; SM70-NEXT: selp.f32 %r28, %r25, %r27, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r28;
+; SM70-NEXT: setp.eq.b32 %p18, %r46, -2147483648;
+; SM70-NEXT: selp.f32 %r29, %r46, %r2, %p18;
+; SM70-NEXT: setp.nan.f32 %p19, %r46, %r2;
+; SM70-NEXT: min.f32 %r30, %r46, %r2;
+; SM70-NEXT: selp.f32 %r31, 0f7FC00000, %r30, %p19;
+; SM70-NEXT: setp.eq.f32 %p20, %r46, %r2;
+; SM70-NEXT: selp.f32 %r32, %r29, %r31, %p20;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r32;
; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
-; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r46;
; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
-; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
-; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM70-NEXT: @%p27 bra $L__BB259_5;
+; SM70-NEXT: setp.ne.b64 %p21, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd31;
+; SM70-NEXT: @%p21 bra $L__BB259_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
-; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
-; SM70-NEXT: setp.eq.b32 %p30, %r3, -2147483648;
-; SM70-NEXT: setp.eq.b32 %p34, %r4, -2147483648;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r47, %rd32;
; SM70-NEXT: $L__BB259_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
-; SM70-NEXT: min.f32 %r39, %r55, %r3;
-; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
-; SM70-NEXT: setp.eq.b32 %p29, %r55, -2147483648;
-; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
-; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
-; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
-; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
-; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
-; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
-; SM70-NEXT: min.f32 %r44, %r56, %r4;
-; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
-; SM70-NEXT: setp.eq.b32 %p33, %r56, -2147483648;
-; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
-; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
-; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
-; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
-; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: setp.eq.b32 %p22, %r47, -2147483648;
+; SM70-NEXT: selp.f32 %r33, %r47, %r3, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r47, %r3;
+; SM70-NEXT: min.f32 %r34, %r47, %r3;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.f32 %p24, %r47, %r3;
+; SM70-NEXT: selp.f32 %r36, %r33, %r35, %p24;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r36;
+; SM70-NEXT: setp.eq.b32 %p25, %r48, -2147483648;
+; SM70-NEXT: selp.f32 %r37, %r48, %r4, %p25;
+; SM70-NEXT: setp.nan.f32 %p26, %r48, %r4;
+; SM70-NEXT: min.f32 %r38, %r48, %r4;
+; SM70-NEXT: selp.f32 %r39, 0f7FC00000, %r38, %p26;
+; SM70-NEXT: setp.eq.f32 %p27, %r48, %r4;
+; SM70-NEXT: selp.f32 %r40, %r37, %r39, %p27;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r40;
; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
-; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r47;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r48;
; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
-; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
-; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM70-NEXT: @%p36 bra $L__BB259_7;
+; SM70-NEXT: setp.ne.b64 %p28, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r47, %rd41;
+; SM70-NEXT: @%p28 bra $L__BB259_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11047,30 +11017,28 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b32 %r<8>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
-; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NEXT: $L__BB260_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r8, %r1;
-; SM70-NEXT: max.f32 %r3, %r8, %r1;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r8, 0;
-; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r8, %r7;
-; SM70-NEXT: setp.ne.b32 %p5, %r2, %r8;
-; SM70-NEXT: mov.b32 %r8, %r2;
-; SM70-NEXT: @%p5 bra $L__BB260_1;
+; SM70-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM70-NEXT: selp.f32 %r3, %r7, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r7, %r1;
+; SM70-NEXT: max.f32 %r4, %r7, %r1;
+; SM70-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r7, %r1;
+; SM70-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
+; SM70-NEXT: setp.ne.b32 %p4, %r2, %r7;
+; SM70-NEXT: mov.b32 %r7, %r2;
+; SM70-NEXT: @%p4 bra $L__BB260_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
@@ -11082,8 +11050,8 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<10>;
-; SM70-NEXT: .reg .b32 %r<15>;
+; SM70-NEXT: .reg .pred %p<8>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -11091,44 +11059,40 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r13, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
-; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd2;
; SM70-NEXT: $L__BB261_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r13, %r1;
-; SM70-NEXT: max.f32 %r3, %r13, %r1;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r13, 0;
-; SM70-NEXT: selp.f32 %r5, %r13, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r1, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: setp.nan.f32 %p5, %r14, %r2;
-; SM70-NEXT: max.f32 %r8, %r14, %r2;
+; SM70-NEXT: setp.eq.b32 %p1, %r11, 0;
+; SM70-NEXT: selp.f32 %r3, %r11, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r11, %r1;
+; SM70-NEXT: max.f32 %r4, %r11, %r1;
+; SM70-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r11, %r1;
+; SM70-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r6;
+; SM70-NEXT: setp.eq.b32 %p4, %r12, 0;
+; SM70-NEXT: selp.f32 %r7, %r12, %r2, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r12, %r2;
+; SM70-NEXT: max.f32 %r8, %r12, %r2;
; SM70-NEXT: selp.f32 %r9, 0f7FC00000, %r8, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r14, 0;
-; SM70-NEXT: selp.f32 %r10, %r14, %r9, %p6;
-; SM70-NEXT: selp.f32 %r11, %r2, %r10, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r9, 0f00000000;
-; SM70-NEXT: selp.f32 %r12, %r11, %r9, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
+; SM70-NEXT: setp.eq.f32 %p6, %r12, %r2;
+; SM70-NEXT: selp.f32 %r10, %r7, %r9, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r13;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r14;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r11;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r12;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r13, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB261_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r11, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB261_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r13, %r14};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -11137,8 +11101,8 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<19>;
-; SM70-NEXT: .reg .b32 %r<29>;
+; SM70-NEXT: .reg .pred %p<15>;
+; SM70-NEXT: .reg .b32 %r<25>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -11146,81 +11110,73 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r25, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r1, 0;
-; SM70-NEXT: setp.eq.b32 %p7, %r2, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
; SM70-NEXT: $L__BB262_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r25, %r1;
-; SM70-NEXT: max.f32 %r5, %r25, %r1;
-; SM70-NEXT: selp.f32 %r6, 0f7FC00000, %r5, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r25, 0;
-; SM70-NEXT: selp.f32 %r7, %r25, %r6, %p2;
-; SM70-NEXT: selp.f32 %r8, %r1, %r7, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r6, 0f00000000;
-; SM70-NEXT: selp.f32 %r9, %r8, %r6, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: setp.nan.f32 %p5, %r26, %r2;
-; SM70-NEXT: max.f32 %r10, %r26, %r2;
+; SM70-NEXT: setp.eq.b32 %p1, %r21, 0;
+; SM70-NEXT: selp.f32 %r5, %r21, %r1, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r21, %r1;
+; SM70-NEXT: max.f32 %r6, %r21, %r1;
+; SM70-NEXT: selp.f32 %r7, 0f7FC00000, %r6, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r21, %r1;
+; SM70-NEXT: selp.f32 %r8, %r5, %r7, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r8;
+; SM70-NEXT: setp.eq.b32 %p4, %r22, 0;
+; SM70-NEXT: selp.f32 %r9, %r22, %r2, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r22, %r2;
+; SM70-NEXT: max.f32 %r10, %r22, %r2;
; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r26, 0;
-; SM70-NEXT: selp.f32 %r12, %r26, %r11, %p6;
-; SM70-NEXT: selp.f32 %r13, %r2, %r12, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r11, 0f00000000;
-; SM70-NEXT: selp.f32 %r14, %r13, %r11, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r14;
+; SM70-NEXT: setp.eq.f32 %p6, %r22, %r2;
+; SM70-NEXT: selp.f32 %r12, %r9, %r11, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r12;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r25;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r26;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r25, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB262_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB262_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r27, %rd12;
-; SM70-NEXT: setp.eq.b32 %p12, %r3, 0;
-; SM70-NEXT: setp.eq.b32 %p16, %r4, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd12;
; SM70-NEXT: $L__BB262_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p10, %r27, %r3;
-; SM70-NEXT: max.f32 %r15, %r27, %r3;
-; SM70-NEXT: selp.f32 %r16, 0f7FC00000, %r15, %p10;
-; SM70-NEXT: setp.eq.b32 %p11, %r27, 0;
-; SM70-NEXT: selp.f32 %r17, %r27, %r16, %p11;
-; SM70-NEXT: selp.f32 %r18, %r3, %r17, %p12;
-; SM70-NEXT: setp.eq.f32 %p13, %r16, 0f00000000;
-; SM70-NEXT: selp.f32 %r19, %r18, %r16, %p13;
-; SM70-NEXT: cvt.u64.u32 %rd13, %r19;
-; SM70-NEXT: setp.nan.f32 %p14, %r28, %r4;
-; SM70-NEXT: max.f32 %r20, %r28, %r4;
-; SM70-NEXT: selp.f32 %r21, 0f7FC00000, %r20, %p14;
-; SM70-NEXT: setp.eq.b32 %p15, %r28, 0;
-; SM70-NEXT: selp.f32 %r22, %r28, %r21, %p15;
-; SM70-NEXT: selp.f32 %r23, %r4, %r22, %p16;
-; SM70-NEXT: setp.eq.f32 %p17, %r21, 0f00000000;
-; SM70-NEXT: selp.f32 %r24, %r23, %r21, %p17;
-; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
+; SM70-NEXT: setp.eq.b32 %p8, %r23, 0;
+; SM70-NEXT: selp.f32 %r13, %r23, %r3, %p8;
+; SM70-NEXT: setp.nan.f32 %p9, %r23, %r3;
+; SM70-NEXT: max.f32 %r14, %r23, %r3;
+; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p9;
+; SM70-NEXT: setp.eq.f32 %p10, %r23, %r3;
+; SM70-NEXT: selp.f32 %r16, %r13, %r15, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r16;
+; SM70-NEXT: setp.eq.b32 %p11, %r24, 0;
+; SM70-NEXT: selp.f32 %r17, %r24, %r4, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r24, %r4;
+; SM70-NEXT: max.f32 %r18, %r24, %r4;
+; SM70-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r24, %r4;
+; SM70-NEXT: selp.f32 %r20, %r17, %r19, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r20;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r27;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r28;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r23;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r24;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r27, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB262_3;
+; SM70-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r23, %rd21;
+; SM70-NEXT: @%p14 bra $L__BB262_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -11229,8 +11185,8 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<37>;
-; SM70-NEXT: .reg .b32 %r<57>;
+; SM70-NEXT: .reg .pred %p<29>;
+; SM70-NEXT: .reg .b32 %r<49>;
; SM70-NEXT: .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -11239,156 +11195,140 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM70-NEXT: setp.eq.b32 %p3, %r5, 0;
-; SM70-NEXT: setp.eq.b32 %p7, %r6, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: $L__BB263_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r49, %r5;
-; SM70-NEXT: max.f32 %r9, %r49, %r5;
-; SM70-NEXT: selp.f32 %r10, 0f7FC00000, %r9, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r49, 0;
-; SM70-NEXT: selp.f32 %r11, %r49, %r10, %p2;
-; SM70-NEXT: selp.f32 %r12, %r5, %r11, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r10, 0f00000000;
-; SM70-NEXT: selp.f32 %r13, %r12, %r10, %p4;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
-; SM70-NEXT: setp.nan.f32 %p5, %r50, %r6;
-; SM70-NEXT: max.f32 %r14, %r50, %r6;
+; SM70-NEXT: setp.eq.b32 %p1, %r41, 0;
+; SM70-NEXT: selp.f32 %r9, %r41, %r5, %p1;
+; SM70-NEXT: setp.nan.f32 %p2, %r41, %r5;
+; SM70-NEXT: max.f32 %r10, %r41, %r5;
+; SM70-NEXT: selp.f32 %r11, 0f7FC00000, %r10, %p2;
+; SM70-NEXT: setp.eq.f32 %p3, %r41, %r5;
+; SM70-NEXT: selp.f32 %r12, %r9, %r11, %p3;
+; SM70-NEXT: cvt.u64.u32 %rd3, %r12;
+; SM70-NEXT: setp.eq.b32 %p4, %r42, 0;
+; SM70-NEXT: selp.f32 %r13, %r42, %r6, %p4;
+; SM70-NEXT: setp.nan.f32 %p5, %r42, %r6;
+; SM70-NEXT: max.f32 %r14, %r42, %r6;
; SM70-NEXT: selp.f32 %r15, 0f7FC00000, %r14, %p5;
-; SM70-NEXT: setp.eq.b32 %p6, %r50, 0;
-; SM70-NEXT: selp.f32 %r16, %r50, %r15, %p6;
-; SM70-NEXT: selp.f32 %r17, %r6, %r16, %p7;
-; SM70-NEXT: setp.eq.f32 %p8, %r15, 0f00000000;
-; SM70-NEXT: selp.f32 %r18, %r17, %r15, %p8;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
+; SM70-NEXT: setp.eq.f32 %p6, %r42, %r6;
+; SM70-NEXT: selp.f32 %r16, %r13, %r15, %p6;
+; SM70-NEXT: cvt.u64.u32 %rd4, %r16;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r49;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r50;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r42;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p9, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r49, %rd11;
-; SM70-NEXT: @%p9 bra $L__BB263_1;
+; SM70-NEXT: setp.ne.b64 %p7, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM70-NEXT: @%p7 bra $L__BB263_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r51, %rd12;
-; SM70-NEXT: setp.eq.b32 %p12, %r7, 0;
-; SM70-NEXT: setp.eq.b32 %p16, %r8, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd12;
; SM70-NEXT: $L__BB263_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p10, %r51, %r7;
-; SM70-NEXT: max.f32 %r19, %r51, %r7;
-; SM70-NEXT: selp.f32 %r20, 0f7FC00000, %r19, %p10;
-; SM70-NEXT: setp.eq.b32 %p11, %r51, 0;
-; SM70-NEXT: selp.f32 %r21, %r51, %r20, %p11;
-; SM70-NEXT: selp.f32 %r22, %r7, %r21, %p12;
-; SM70-NEXT: setp.eq.f32 %p13, %r20, 0f00000000;
-; SM70-NEXT: selp.f32 %r23, %r22, %r20, %p13;
-; SM70-NEXT: cvt.u64.u32 %rd13, %r23;
-; SM70-NEXT: setp.nan.f32 %p14, %r52, %r8;
-; SM70-NEXT: max.f32 %r24, %r52, %r8;
-; SM70-NEXT: selp.f32 %r25, 0f7FC00000, %r24, %p14;
-; SM70-NEXT: setp.eq.b32 %p15, %r52, 0;
-; SM70-NEXT: selp.f32 %r26, %r52, %r25, %p15;
-; SM70-NEXT: selp.f32 %r27, %r8, %r26, %p16;
-; SM70-NEXT: setp.eq.f32 %p17, %r25, 0f00000000;
-; SM70-NEXT: selp.f32 %r28, %r27, %r25, %p17;
-; SM70-NEXT: cvt.u64.u32 %rd14, %r28;
+; SM70-NEXT: setp.eq.b32 %p8, %r43, 0;
+; SM70-NEXT: selp.f32 %r17, %r43, %r7, %p8;
+; SM70-NEXT: setp.nan.f32 %p9, %r43, %r7;
+; SM70-NEXT: max.f32 %r18, %r43, %r7;
+; SM70-NEXT: selp.f32 %r19, 0f7FC00000, %r18, %p9;
+; SM70-NEXT: setp.eq.f32 %p10, %r43, %r7;
+; SM70-NEXT: selp.f32 %r20, %r17, %r19, %p10;
+; SM70-NEXT: cvt.u64.u32 %rd13, %r20;
+; SM70-NEXT: setp.eq.b32 %p11, %r44, 0;
+; SM70-NEXT: selp.f32 %r21, %r44, %r8, %p11;
+; SM70-NEXT: setp.nan.f32 %p12, %r44, %r8;
+; SM70-NEXT: max.f32 %r22, %r44, %r8;
+; SM70-NEXT: selp.f32 %r23, 0f7FC00000, %r22, %p12;
+; SM70-NEXT: setp.eq.f32 %p13, %r44, %r8;
+; SM70-NEXT: selp.f32 %r24, %r21, %r23, %p13;
+; SM70-NEXT: cvt.u64.u32 %rd14, %r24;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r51;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r52;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r44;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p18, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r52}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r51, %rd21;
-; SM70-NEXT: @%p18 bra $L__BB263_3;
+; SM70-NEXT: setp.ne.b64 %p14, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM70-NEXT: @%p14 bra $L__BB263_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd22; }
-; SM70-NEXT: cvt.u32.u64 %r53, %rd22;
-; SM70-NEXT: setp.eq.b32 %p21, %r1, 0;
-; SM70-NEXT: setp.eq.b32 %p25, %r2, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd22;
; SM70-NEXT: $L__BB263_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p19, %r53, %r1;
-; SM70-NEXT: max.f32 %r29, %r53, %r1;
-; SM70-NEXT: selp.f32 %r30, 0f7FC00000, %r29, %p19;
-; SM70-NEXT: setp.eq.b32 %p20, %r53, 0;
-; SM70-NEXT: selp.f32 %r31, %r53, %r30, %p20;
-; SM70-NEXT: selp.f32 %r32, %r1, %r31, %p21;
-; SM70-NEXT: setp.eq.f32 %p22, %r30, 0f00000000;
-; SM70-NEXT: selp.f32 %r33, %r32, %r30, %p22;
-; SM70-NEXT: cvt.u64.u32 %rd23, %r33;
-; SM70-NEXT: setp.nan.f32 %p23, %r54, %r2;
-; SM70-NEXT: max.f32 %r34, %r54, %r2;
-; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
-; SM70-NEXT: setp.eq.b32 %p24, %r54, 0;
-; SM70-NEXT: selp.f32 %r36, %r54, %r35, %p24;
-; SM70-NEXT: selp.f32 %r37, %r2, %r36, %p25;
-; SM70-NEXT: setp.eq.f32 %p26, %r35, 0f00000000;
-; SM70-NEXT: selp.f32 %r38, %r37, %r35, %p26;
-; SM70-NEXT: cvt.u64.u32 %rd24, %r38;
+; SM70-NEXT: setp.eq.b32 %p15, %r45, 0;
+; SM70-NEXT: selp.f32 %r25, %r45, %r1, %p15;
+; SM70-NEXT: setp.nan.f32 %p16, %r45, %r1;
+; SM70-NEXT: max.f32 %r26, %r45, %r1;
+; SM70-NEXT: selp.f32 %r27, 0f7FC00000, %r26, %p16;
+; SM70-NEXT: setp.eq.f32 %p17, %r45, %r1;
+; SM70-NEXT: selp.f32 %r28, %r25, %r27, %p17;
+; SM70-NEXT: cvt.u64.u32 %rd23, %r28;
+; SM70-NEXT: setp.eq.b32 %p18, %r46, 0;
+; SM70-NEXT: selp.f32 %r29, %r46, %r2, %p18;
+; SM70-NEXT: setp.nan.f32 %p19, %r46, %r2;
+; SM70-NEXT: max.f32 %r30, %r46, %r2;
+; SM70-NEXT: selp.f32 %r31, 0f7FC00000, %r30, %p19;
+; SM70-NEXT: setp.eq.f32 %p20, %r46, %r2;
+; SM70-NEXT: selp.f32 %r32, %r29, %r31, %p20;
+; SM70-NEXT: cvt.u64.u32 %rd24, %r32;
; SM70-NEXT: shl.b64 %rd25, %rd24, 32;
; SM70-NEXT: or.b64 %rd26, %rd23, %rd25;
-; SM70-NEXT: cvt.u64.u32 %rd27, %r53;
-; SM70-NEXT: cvt.u64.u32 %rd28, %r54;
+; SM70-NEXT: cvt.u64.u32 %rd27, %r45;
+; SM70-NEXT: cvt.u64.u32 %rd28, %r46;
; SM70-NEXT: shl.b64 %rd29, %rd28, 32;
; SM70-NEXT: or.b64 %rd30, %rd27, %rd29;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
-; SM70-NEXT: setp.ne.b64 %p27, %rd31, %rd30;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r54}, %rd31; }
-; SM70-NEXT: cvt.u32.u64 %r53, %rd31;
-; SM70-NEXT: @%p27 bra $L__BB263_5;
+; SM70-NEXT: setp.ne.b64 %p21, %rd31, %rd30;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
+; SM70-NEXT: cvt.u32.u64 %r45, %rd31;
+; SM70-NEXT: @%p21 bra $L__BB263_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd32; }
-; SM70-NEXT: cvt.u32.u64 %r55, %rd32;
-; SM70-NEXT: setp.eq.b32 %p30, %r3, 0;
-; SM70-NEXT: setp.eq.b32 %p34, %r4, 0;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
+; SM70-NEXT: cvt.u32.u64 %r47, %rd32;
; SM70-NEXT: $L__BB263_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p28, %r55, %r3;
-; SM70-NEXT: max.f32 %r39, %r55, %r3;
-; SM70-NEXT: selp.f32 %r40, 0f7FC00000, %r39, %p28;
-; SM70-NEXT: setp.eq.b32 %p29, %r55, 0;
-; SM70-NEXT: selp.f32 %r41, %r55, %r40, %p29;
-; SM70-NEXT: selp.f32 %r42, %r3, %r41, %p30;
-; SM70-NEXT: setp.eq.f32 %p31, %r40, 0f00000000;
-; SM70-NEXT: selp.f32 %r43, %r42, %r40, %p31;
-; SM70-NEXT: cvt.u64.u32 %rd33, %r43;
-; SM70-NEXT: setp.nan.f32 %p32, %r56, %r4;
-; SM70-NEXT: max.f32 %r44, %r56, %r4;
-; SM70-NEXT: selp.f32 %r45, 0f7FC00000, %r44, %p32;
-; SM70-NEXT: setp.eq.b32 %p33, %r56, 0;
-; SM70-NEXT: selp.f32 %r46, %r56, %r45, %p33;
-; SM70-NEXT: selp.f32 %r47, %r4, %r46, %p34;
-; SM70-NEXT: setp.eq.f32 %p35, %r45, 0f00000000;
-; SM70-NEXT: selp.f32 %r48, %r47, %r45, %p35;
-; SM70-NEXT: cvt.u64.u32 %rd34, %r48;
+; SM70-NEXT: setp.eq.b32 %p22, %r47, 0;
+; SM70-NEXT: selp.f32 %r33, %r47, %r3, %p22;
+; SM70-NEXT: setp.nan.f32 %p23, %r47, %r3;
+; SM70-NEXT: max.f32 %r34, %r47, %r3;
+; SM70-NEXT: selp.f32 %r35, 0f7FC00000, %r34, %p23;
+; SM70-NEXT: setp.eq.f32 %p24, %r47, %r3;
+; SM70-NEXT: selp.f32 %r36, %r33, %r35, %p24;
+; SM70-NEXT: cvt.u64.u32 %rd33, %r36;
+; SM70-NEXT: setp.eq.b32 %p25, %r48, 0;
+; SM70-NEXT: selp.f32 %r37, %r48, %r4, %p25;
+; SM70-NEXT: setp.nan.f32 %p26, %r48, %r4;
+; SM70-NEXT: max.f32 %r38, %r48, %r4;
+; SM70-NEXT: selp.f32 %r39, 0f7FC00000, %r38, %p26;
+; SM70-NEXT: setp.eq.f32 %p27, %r48, %r4;
+; SM70-NEXT: selp.f32 %r40, %r37, %r39, %p27;
+; SM70-NEXT: cvt.u64.u32 %rd34, %r40;
; SM70-NEXT: shl.b64 %rd35, %rd34, 32;
; SM70-NEXT: or.b64 %rd36, %rd33, %rd35;
-; SM70-NEXT: cvt.u64.u32 %rd37, %r55;
-; SM70-NEXT: cvt.u64.u32 %rd38, %r56;
+; SM70-NEXT: cvt.u64.u32 %rd37, %r47;
+; SM70-NEXT: cvt.u64.u32 %rd38, %r48;
; SM70-NEXT: shl.b64 %rd39, %rd38, 32;
; SM70-NEXT: or.b64 %rd40, %rd37, %rd39;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
-; SM70-NEXT: setp.ne.b64 %p36, %rd41, %rd40;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r56}, %rd41; }
-; SM70-NEXT: cvt.u32.u64 %r55, %rd41;
-; SM70-NEXT: @%p36 bra $L__BB263_7;
+; SM70-NEXT: setp.ne.b64 %p28, %rd41, %rd40;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
+; SM70-NEXT: cvt.u32.u64 %r47, %rd41;
+; SM70-NEXT: @%p28 bra $L__BB263_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r53, %r54, %r55, %r56};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r49, %r50, %r51, %r52};
+; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -12117,29 +12057,27 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v1f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
-; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM70-NEXT: $L__BB280_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM70-NEXT: min.f64 %rd4, %rd9, %rd1;
-; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM70-NEXT: mov.b64 %rd9, %rd2;
-; SM70-NEXT: @%p5 bra $L__BB280_1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM70-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM70-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd2;
+; SM70-NEXT: @%p4 bra $L__BB280_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12151,46 +12089,42 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<11>;
-; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<16>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v2f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
-; SM70-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT: $L__BB281_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
-; SM70-NEXT: min.f64 %rd6, %rd16, %rd1;
-; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd16, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
-; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
-; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
-; SM70-NEXT: mov.b64 %rd16, %rd3;
-; SM70-NEXT: @%p5 bra $L__BB281_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd2, -9223372036854775808;
+; SM70-NEXT: setp.eq.b64 %p1, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM70-NEXT: min.f64 %rd7, %rd14, %rd1;
+; SM70-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
+; SM70-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM70-NEXT: mov.b64 %rd14, %rd3;
+; SM70-NEXT: @%p4 bra $L__BB281_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
; SM70-NEXT: $L__BB281_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
-; SM70-NEXT: min.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: setp.eq.b64 %p5, %rd15, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM70-NEXT: min.f64 %rd11, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd17, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
-; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
-; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
-; SM70-NEXT: mov.b64 %rd17, %rd4;
-; SM70-NEXT: @%p10 bra $L__BB281_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
+; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
+; SM70-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
+; SM70-NEXT: mov.b64 %rd15, %rd4;
+; SM70-NEXT: @%p8 bra $L__BB281_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -12202,85 +12136,77 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v4f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<21>;
-; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<30>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
-; SM70-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT: $L__BB282_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd30;
-; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
-; SM70-NEXT: min.f64 %rd10, %rd5, %rd3;
-; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd5, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
-; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
-; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM70-NEXT: @%p5 bra $L__BB282_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd4, -9223372036854775808;
+; SM70-NEXT: mov.b64 %rd5, %rd26;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM70-NEXT: min.f64 %rd11, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
+; SM70-NEXT: @%p4 bra $L__BB282_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
; SM70-NEXT: $L__BB282_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: mov.b64 %rd6, %rd27;
+; SM70-NEXT: setp.eq.b64 %p5, %rd6, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
; SM70-NEXT: min.f64 %rd15, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
-; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
-; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM70-NEXT: @%p10 bra $L__BB282_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
+; SM70-NEXT: @%p8 bra $L__BB282_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
-; SM70-NEXT: setp.eq.b64 %p13, %rd1, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
; SM70-NEXT: $L__BB282_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
-; SM70-NEXT: min.f64 %rd20, %rd32, %rd1;
-; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
-; SM70-NEXT: setp.eq.b64 %p12, %rd32, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
-; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
-; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
-; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
-; SM70-NEXT: mov.b64 %rd32, %rd7;
-; SM70-NEXT: @%p15 bra $L__BB282_5;
+; SM70-NEXT: setp.eq.b64 %p9, %rd28, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM70-NEXT: min.f64 %rd19, %rd28, %rd1;
+; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
+; SM70-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
+; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM70-NEXT: mov.b64 %rd28, %rd7;
+; SM70-NEXT: @%p12 bra $L__BB282_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
-; SM70-NEXT: setp.eq.b64 %p18, %rd2, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
; SM70-NEXT: $L__BB282_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
-; SM70-NEXT: min.f64 %rd25, %rd33, %rd2;
-; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
-; SM70-NEXT: setp.eq.b64 %p17, %rd33, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
-; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
-; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
-; SM70-NEXT: mov.b64 %rd33, %rd8;
-; SM70-NEXT: @%p20 bra $L__BB282_7;
+; SM70-NEXT: setp.eq.b64 %p13, %rd29, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM70-NEXT: min.f64 %rd23, %rd29, %rd2;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
+; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
+; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
+; SM70-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
+; SM70-NEXT: mov.b64 %rd29, %rd8;
+; SM70-NEXT: @%p16 bra $L__BB282_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12289,8 +12215,8 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v8f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<41>;
-; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-NEXT: .reg .pred %p<33>;
+; SM70-NEXT: .reg .b64 %rd<58>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
@@ -12299,147 +12225,131 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
-; SM70-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT: $L__BB283_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd58;
-; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
-; SM70-NEXT: min.f64 %rd18, %rd7, %rd5;
-; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd7, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
-; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
-; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM70-NEXT: @%p5 bra $L__BB283_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd6, -9223372036854775808;
+; SM70-NEXT: mov.b64 %rd7, %rd50;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM70-NEXT: min.f64 %rd19, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
+; SM70-NEXT: @%p4 bra $L__BB283_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
; SM70-NEXT: $L__BB283_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: mov.b64 %rd8, %rd51;
+; SM70-NEXT: setp.eq.b64 %p5, %rd8, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
; SM70-NEXT: min.f64 %rd23, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd8, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
-; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
-; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM70-NEXT: @%p10 bra $L__BB283_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB283_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
-; SM70-NEXT: setp.eq.b64 %p13, %rd3, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
; SM70-NEXT: $L__BB283_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd60;
-; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
-; SM70-NEXT: min.f64 %rd28, %rd9, %rd3;
-; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
-; SM70-NEXT: setp.eq.b64 %p12, %rd9, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
-; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
-; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
-; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM70-NEXT: @%p15 bra $L__BB283_5;
+; SM70-NEXT: mov.b64 %rd9, %rd52;
+; SM70-NEXT: setp.eq.b64 %p9, %rd9, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM70-NEXT: min.f64 %rd27, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
+; SM70-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
+; SM70-NEXT: @%p12 bra $L__BB283_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
-; SM70-NEXT: setp.eq.b64 %p18, %rd4, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
; SM70-NEXT: $L__BB283_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd10, %rd61;
-; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
-; SM70-NEXT: min.f64 %rd33, %rd10, %rd4;
-; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
-; SM70-NEXT: setp.eq.b64 %p17, %rd10, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
-; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
-; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM70-NEXT: @%p20 bra $L__BB283_7;
+; SM70-NEXT: mov.b64 %rd10, %rd53;
+; SM70-NEXT: setp.eq.b64 %p13, %rd10, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM70-NEXT: min.f64 %rd31, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
+; SM70-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
+; SM70-NEXT: @%p16 bra $L__BB283_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
-; SM70-NEXT: setp.eq.b64 %p23, %rd1, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
; SM70-NEXT: $L__BB283_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd62;
-; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
-; SM70-NEXT: min.f64 %rd38, %rd13, %rd1;
-; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
-; SM70-NEXT: setp.eq.b64 %p22, %rd13, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
-; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
-; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
-; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM70-NEXT: @%p25 bra $L__BB283_9;
+; SM70-NEXT: mov.b64 %rd13, %rd54;
+; SM70-NEXT: setp.eq.b64 %p17, %rd13, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM70-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM70-NEXT: min.f64 %rd35, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
+; SM70-NEXT: @%p20 bra $L__BB283_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
-; SM70-NEXT: setp.eq.b64 %p28, %rd2, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
; SM70-NEXT: $L__BB283_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd14, %rd63;
-; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
-; SM70-NEXT: min.f64 %rd43, %rd14, %rd2;
-; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM70-NEXT: setp.eq.b64 %p27, %rd14, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
-; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
-; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
-; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM70-NEXT: @%p30 bra $L__BB283_11;
+; SM70-NEXT: mov.b64 %rd14, %rd55;
+; SM70-NEXT: setp.eq.b64 %p21, %rd14, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM70-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM70-NEXT: min.f64 %rd39, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
+; SM70-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
+; SM70-NEXT: @%p24 bra $L__BB283_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
-; SM70-NEXT: setp.eq.b64 %p33, %rd11, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
; SM70-NEXT: $L__BB283_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
-; SM70-NEXT: min.f64 %rd48, %rd64, %rd11;
-; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
-; SM70-NEXT: setp.eq.b64 %p32, %rd64, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
-; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
-; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
-; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
-; SM70-NEXT: mov.b64 %rd64, %rd15;
-; SM70-NEXT: @%p35 bra $L__BB283_13;
+; SM70-NEXT: setp.eq.b64 %p25, %rd56, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM70-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM70-NEXT: min.f64 %rd43, %rd56, %rd11;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
+; SM70-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM70-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM70-NEXT: mov.b64 %rd56, %rd15;
+; SM70-NEXT: @%p28 bra $L__BB283_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
-; SM70-NEXT: setp.eq.b64 %p38, %rd12, -9223372036854775808;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
; SM70-NEXT: $L__BB283_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
-; SM70-NEXT: min.f64 %rd53, %rd65, %rd12;
-; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
-; SM70-NEXT: setp.eq.b64 %p37, %rd65, -9223372036854775808;
-; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
-; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
-; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
-; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
-; SM70-NEXT: mov.b64 %rd65, %rd16;
-; SM70-NEXT: @%p40 bra $L__BB283_15;
+; SM70-NEXT: setp.eq.b64 %p29, %rd57, -9223372036854775808;
+; SM70-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM70-NEXT: min.f64 %rd47, %rd57, %rd12;
+; SM70-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
+; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
+; SM70-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
+; SM70-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
+; SM70-NEXT: mov.b64 %rd57, %rd16;
+; SM70-NEXT: @%p32 bra $L__BB283_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12448,29 +12358,27 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v1f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b64 %rd<10>;
+; SM70-NEXT: .reg .pred %p<5>;
+; SM70-NEXT: .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
-; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM70-NEXT: $L__BB284_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM70-NEXT: max.f64 %rd4, %rd9, %rd1;
-; SM70-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd9, 0;
-; SM70-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM70-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM70-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM70-NEXT: mov.b64 %rd9, %rd2;
-; SM70-NEXT: @%p5 bra $L__BB284_1;
+; SM70-NEXT: setp.eq.b64 %p1, %rd8, 0;
+; SM70-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM70-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM70-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM70-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM70-NEXT: mov.b64 %rd8, %rd2;
+; SM70-NEXT: @%p4 bra $L__BB284_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12482,46 +12390,42 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<11>;
-; SM70-NEXT: .reg .b64 %rd<18>;
+; SM70-NEXT: .reg .pred %p<9>;
+; SM70-NEXT: .reg .b64 %rd<16>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd16, [%rd5];
-; SM70-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT: $L__BB285_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p1, %rd16, %rd1;
-; SM70-NEXT: max.f64 %rd6, %rd16, %rd1;
-; SM70-NEXT: selp.f64 %rd7, 0d7FF8000000000000, %rd6, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd16, 0;
-; SM70-NEXT: selp.f64 %rd8, %rd16, %rd7, %p2;
-; SM70-NEXT: selp.f64 %rd9, %rd1, %rd8, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd7, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd10, %rd9, %rd7, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd16, %rd10;
-; SM70-NEXT: setp.ne.b64 %p5, %rd3, %rd16;
-; SM70-NEXT: mov.b64 %rd16, %rd3;
-; SM70-NEXT: @%p5 bra $L__BB285_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd17, [%rd5+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd2, 0;
+; SM70-NEXT: setp.eq.b64 %p1, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd6, %rd14, %rd1, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd14, %rd1;
+; SM70-NEXT: max.f64 %rd7, %rd14, %rd1;
+; SM70-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd14, %rd1;
+; SM70-NEXT: selp.f64 %rd9, %rd6, %rd8, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd3, [%rd5], %rd14, %rd9;
+; SM70-NEXT: setp.ne.b64 %p4, %rd3, %rd14;
+; SM70-NEXT: mov.b64 %rd14, %rd3;
+; SM70-NEXT: @%p4 bra $L__BB285_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
; SM70-NEXT: $L__BB285_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p6, %rd17, %rd2;
-; SM70-NEXT: max.f64 %rd11, %rd17, %rd2;
+; SM70-NEXT: setp.eq.b64 %p5, %rd15, 0;
+; SM70-NEXT: selp.f64 %rd10, %rd15, %rd2, %p5;
+; SM70-NEXT: setp.nan.f64 %p6, %rd15, %rd2;
+; SM70-NEXT: max.f64 %rd11, %rd15, %rd2;
; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd17, 0;
-; SM70-NEXT: selp.f64 %rd13, %rd17, %rd12, %p7;
-; SM70-NEXT: selp.f64 %rd14, %rd2, %rd13, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd12, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd15, %rd14, %rd12, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd17, %rd15;
-; SM70-NEXT: setp.ne.b64 %p10, %rd4, %rd17;
-; SM70-NEXT: mov.b64 %rd17, %rd4;
-; SM70-NEXT: @%p10 bra $L__BB285_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd15, %rd2;
+; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
+; SM70-NEXT: setp.ne.b64 %p8, %rd4, %rd15;
+; SM70-NEXT: mov.b64 %rd15, %rd4;
+; SM70-NEXT: @%p8 bra $L__BB285_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
@@ -12533,85 +12437,77 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v4f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<21>;
-; SM70-NEXT: .reg .b64 %rd<34>;
+; SM70-NEXT: .reg .pred %p<17>;
+; SM70-NEXT: .reg .b64 %rd<30>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd30, [%rd9];
-; SM70-NEXT: setp.eq.b64 %p3, %rd3, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT: $L__BB286_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd5, %rd30;
-; SM70-NEXT: setp.nan.f64 %p1, %rd5, %rd3;
-; SM70-NEXT: max.f64 %rd10, %rd5, %rd3;
-; SM70-NEXT: selp.f64 %rd11, 0d7FF8000000000000, %rd10, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd5, 0;
-; SM70-NEXT: selp.f64 %rd12, %rd5, %rd11, %p2;
-; SM70-NEXT: selp.f64 %rd13, %rd3, %rd12, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd11, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd14, %rd13, %rd11, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd30, [%rd9], %rd5, %rd14;
-; SM70-NEXT: setp.ne.b64 %p5, %rd30, %rd5;
-; SM70-NEXT: @%p5 bra $L__BB286_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd31, [%rd9+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd4, 0;
+; SM70-NEXT: mov.b64 %rd5, %rd26;
+; SM70-NEXT: setp.eq.b64 %p1, %rd5, 0;
+; SM70-NEXT: selp.f64 %rd10, %rd5, %rd3, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd5, %rd3;
+; SM70-NEXT: max.f64 %rd11, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd5, %rd3;
+; SM70-NEXT: selp.f64 %rd13, %rd10, %rd12, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd5, %rd13;
+; SM70-NEXT: setp.ne.b64 %p4, %rd26, %rd5;
+; SM70-NEXT: @%p4 bra $L__BB286_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
; SM70-NEXT: $L__BB286_3: // %atomicrmw.start5
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd6, %rd31;
+; SM70-NEXT: mov.b64 %rd6, %rd27;
+; SM70-NEXT: setp.eq.b64 %p5, %rd6, 0;
+; SM70-NEXT: selp.f64 %rd14, %rd6, %rd4, %p5;
; SM70-NEXT: setp.nan.f64 %p6, %rd6, %rd4;
; SM70-NEXT: max.f64 %rd15, %rd6, %rd4;
; SM70-NEXT: selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd6, 0;
-; SM70-NEXT: selp.f64 %rd17, %rd6, %rd16, %p7;
-; SM70-NEXT: selp.f64 %rd18, %rd4, %rd17, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd16, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd19, %rd18, %rd16, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd31, [%rd9+8], %rd6, %rd19;
-; SM70-NEXT: setp.ne.b64 %p10, %rd31, %rd6;
-; SM70-NEXT: @%p10 bra $L__BB286_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd6, %rd4;
+; SM70-NEXT: selp.f64 %rd17, %rd14, %rd16, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd6, %rd17;
+; SM70-NEXT: setp.ne.b64 %p8, %rd27, %rd6;
+; SM70-NEXT: @%p8 bra $L__BB286_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end4
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd32, [%rd9+16];
-; SM70-NEXT: setp.eq.b64 %p13, %rd1, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
; SM70-NEXT: $L__BB286_5: // %atomicrmw.start15
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p11, %rd32, %rd1;
-; SM70-NEXT: max.f64 %rd20, %rd32, %rd1;
-; SM70-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p11;
-; SM70-NEXT: setp.eq.b64 %p12, %rd32, 0;
-; SM70-NEXT: selp.f64 %rd22, %rd32, %rd21, %p12;
-; SM70-NEXT: selp.f64 %rd23, %rd1, %rd22, %p13;
-; SM70-NEXT: setp.eq.f64 %p14, %rd21, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd24, %rd23, %rd21, %p14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd32, %rd24;
-; SM70-NEXT: setp.ne.b64 %p15, %rd7, %rd32;
-; SM70-NEXT: mov.b64 %rd32, %rd7;
-; SM70-NEXT: @%p15 bra $L__BB286_5;
+; SM70-NEXT: setp.eq.b64 %p9, %rd28, 0;
+; SM70-NEXT: selp.f64 %rd18, %rd28, %rd1, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd28, %rd1;
+; SM70-NEXT: max.f64 %rd19, %rd28, %rd1;
+; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
+; SM70-NEXT: setp.eq.f64 %p11, %rd28, %rd1;
+; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd7, [%rd9+16], %rd28, %rd21;
+; SM70-NEXT: setp.ne.b64 %p12, %rd7, %rd28;
+; SM70-NEXT: mov.b64 %rd28, %rd7;
+; SM70-NEXT: @%p12 bra $L__BB286_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end14
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd33, [%rd9+24];
-; SM70-NEXT: setp.eq.b64 %p18, %rd2, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
; SM70-NEXT: $L__BB286_7: // %atomicrmw.start20
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p16, %rd33, %rd2;
-; SM70-NEXT: max.f64 %rd25, %rd33, %rd2;
-; SM70-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p16;
-; SM70-NEXT: setp.eq.b64 %p17, %rd33, 0;
-; SM70-NEXT: selp.f64 %rd27, %rd33, %rd26, %p17;
-; SM70-NEXT: selp.f64 %rd28, %rd2, %rd27, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd26, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd29, %rd28, %rd26, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd33, %rd29;
-; SM70-NEXT: setp.ne.b64 %p20, %rd8, %rd33;
-; SM70-NEXT: mov.b64 %rd33, %rd8;
-; SM70-NEXT: @%p20 bra $L__BB286_7;
+; SM70-NEXT: setp.eq.b64 %p13, %rd29, 0;
+; SM70-NEXT: selp.f64 %rd22, %rd29, %rd2, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd29, %rd2;
+; SM70-NEXT: max.f64 %rd23, %rd29, %rd2;
+; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
+; SM70-NEXT: setp.eq.f64 %p15, %rd29, %rd2;
+; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
+; SM70-NEXT: setp.ne.b64 %p16, %rd8, %rd29;
+; SM70-NEXT: mov.b64 %rd29, %rd8;
+; SM70-NEXT: @%p16 bra $L__BB286_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end19
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd30, %rd31};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12620,8 +12516,8 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v8f64_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<41>;
-; SM70-NEXT: .reg .b64 %rd<66>;
+; SM70-NEXT: .reg .pred %p<33>;
+; SM70-NEXT: .reg .b64 %rd<58>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
@@ -12630,147 +12526,131 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd58, [%rd17];
-; SM70-NEXT: setp.eq.b64 %p3, %rd5, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT: $L__BB287_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd7, %rd58;
-; SM70-NEXT: setp.nan.f64 %p1, %rd7, %rd5;
-; SM70-NEXT: max.f64 %rd18, %rd7, %rd5;
-; SM70-NEXT: selp.f64 %rd19, 0d7FF8000000000000, %rd18, %p1;
-; SM70-NEXT: setp.eq.b64 %p2, %rd7, 0;
-; SM70-NEXT: selp.f64 %rd20, %rd7, %rd19, %p2;
-; SM70-NEXT: selp.f64 %rd21, %rd5, %rd20, %p3;
-; SM70-NEXT: setp.eq.f64 %p4, %rd19, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd22, %rd21, %rd19, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd58, [%rd17], %rd7, %rd22;
-; SM70-NEXT: setp.ne.b64 %p5, %rd58, %rd7;
-; SM70-NEXT: @%p5 bra $L__BB287_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd59, [%rd17+8];
-; SM70-NEXT: setp.eq.b64 %p8, %rd6, 0;
+; SM70-NEXT: mov.b64 %rd7, %rd50;
+; SM70-NEXT: setp.eq.b64 %p1, %rd7, 0;
+; SM70-NEXT: selp.f64 %rd18, %rd7, %rd5, %p1;
+; SM70-NEXT: setp.nan.f64 %p2, %rd7, %rd5;
+; SM70-NEXT: max.f64 %rd19, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
+; SM70-NEXT: setp.eq.f64 %p3, %rd7, %rd5;
+; SM70-NEXT: selp.f64 %rd21, %rd18, %rd20, %p3;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd7, %rd21;
+; SM70-NEXT: setp.ne.b64 %p4, %rd50, %rd7;
+; SM70-NEXT: @%p4 bra $L__BB287_1;
+; SM70-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
; SM70-NEXT: $L__BB287_3: // %atomicrmw.start9
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd8, %rd59;
+; SM70-NEXT: mov.b64 %rd8, %rd51;
+; SM70-NEXT: setp.eq.b64 %p5, %rd8, 0;
+; SM70-NEXT: selp.f64 %rd22, %rd8, %rd6, %p5;
; SM70-NEXT: setp.nan.f64 %p6, %rd8, %rd6;
; SM70-NEXT: max.f64 %rd23, %rd8, %rd6;
; SM70-NEXT: selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
-; SM70-NEXT: setp.eq.b64 %p7, %rd8, 0;
-; SM70-NEXT: selp.f64 %rd25, %rd8, %rd24, %p7;
-; SM70-NEXT: selp.f64 %rd26, %rd6, %rd25, %p8;
-; SM70-NEXT: setp.eq.f64 %p9, %rd24, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd27, %rd26, %rd24, %p9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd59, [%rd17+8], %rd8, %rd27;
-; SM70-NEXT: setp.ne.b64 %p10, %rd59, %rd8;
-; SM70-NEXT: @%p10 bra $L__BB287_3;
+; SM70-NEXT: setp.eq.f64 %p7, %rd8, %rd6;
+; SM70-NEXT: selp.f64 %rd25, %rd22, %rd24, %p7;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd8, %rd25;
+; SM70-NEXT: setp.ne.b64 %p8, %rd51, %rd8;
+; SM70-NEXT: @%p8 bra $L__BB287_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end8
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd60, [%rd17+16];
-; SM70-NEXT: setp.eq.b64 %p13, %rd3, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
; SM70-NEXT: $L__BB287_5: // %atomicrmw.start19
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd9, %rd60;
-; SM70-NEXT: setp.nan.f64 %p11, %rd9, %rd3;
-; SM70-NEXT: max.f64 %rd28, %rd9, %rd3;
-; SM70-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p11;
-; SM70-NEXT: setp.eq.b64 %p12, %rd9, 0;
-; SM70-NEXT: selp.f64 %rd30, %rd9, %rd29, %p12;
-; SM70-NEXT: selp.f64 %rd31, %rd3, %rd30, %p13;
-; SM70-NEXT: setp.eq.f64 %p14, %rd29, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd32, %rd31, %rd29, %p14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd60, [%rd17+16], %rd9, %rd32;
-; SM70-NEXT: setp.ne.b64 %p15, %rd60, %rd9;
-; SM70-NEXT: @%p15 bra $L__BB287_5;
+; SM70-NEXT: mov.b64 %rd9, %rd52;
+; SM70-NEXT: setp.eq.b64 %p9, %rd9, 0;
+; SM70-NEXT: selp.f64 %rd26, %rd9, %rd3, %p9;
+; SM70-NEXT: setp.nan.f64 %p10, %rd9, %rd3;
+; SM70-NEXT: max.f64 %rd27, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
+; SM70-NEXT: setp.eq.f64 %p11, %rd9, %rd3;
+; SM70-NEXT: selp.f64 %rd29, %rd26, %rd28, %p11;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd9, %rd29;
+; SM70-NEXT: setp.ne.b64 %p12, %rd52, %rd9;
+; SM70-NEXT: @%p12 bra $L__BB287_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end18
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd61, [%rd17+24];
-; SM70-NEXT: setp.eq.b64 %p18, %rd4, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
; SM70-NEXT: $L__BB287_7: // %atomicrmw.start24
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd10, %rd61;
-; SM70-NEXT: setp.nan.f64 %p16, %rd10, %rd4;
-; SM70-NEXT: max.f64 %rd33, %rd10, %rd4;
-; SM70-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p16;
-; SM70-NEXT: setp.eq.b64 %p17, %rd10, 0;
-; SM70-NEXT: selp.f64 %rd35, %rd10, %rd34, %p17;
-; SM70-NEXT: selp.f64 %rd36, %rd4, %rd35, %p18;
-; SM70-NEXT: setp.eq.f64 %p19, %rd34, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd37, %rd36, %rd34, %p19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd61, [%rd17+24], %rd10, %rd37;
-; SM70-NEXT: setp.ne.b64 %p20, %rd61, %rd10;
-; SM70-NEXT: @%p20 bra $L__BB287_7;
+; SM70-NEXT: mov.b64 %rd10, %rd53;
+; SM70-NEXT: setp.eq.b64 %p13, %rd10, 0;
+; SM70-NEXT: selp.f64 %rd30, %rd10, %rd4, %p13;
+; SM70-NEXT: setp.nan.f64 %p14, %rd10, %rd4;
+; SM70-NEXT: max.f64 %rd31, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
+; SM70-NEXT: setp.eq.f64 %p15, %rd10, %rd4;
+; SM70-NEXT: selp.f64 %rd33, %rd30, %rd32, %p15;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd10, %rd33;
+; SM70-NEXT: setp.ne.b64 %p16, %rd53, %rd10;
+; SM70-NEXT: @%p16 bra $L__BB287_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end23
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd62, [%rd17+32];
-; SM70-NEXT: setp.eq.b64 %p23, %rd1, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
; SM70-NEXT: $L__BB287_9: // %atomicrmw.start38
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd13, %rd62;
-; SM70-NEXT: setp.nan.f64 %p21, %rd13, %rd1;
-; SM70-NEXT: max.f64 %rd38, %rd13, %rd1;
-; SM70-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p21;
-; SM70-NEXT: setp.eq.b64 %p22, %rd13, 0;
-; SM70-NEXT: selp.f64 %rd40, %rd13, %rd39, %p22;
-; SM70-NEXT: selp.f64 %rd41, %rd1, %rd40, %p23;
-; SM70-NEXT: setp.eq.f64 %p24, %rd39, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd42, %rd41, %rd39, %p24;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd62, [%rd17+32], %rd13, %rd42;
-; SM70-NEXT: setp.ne.b64 %p25, %rd62, %rd13;
-; SM70-NEXT: @%p25 bra $L__BB287_9;
+; SM70-NEXT: mov.b64 %rd13, %rd54;
+; SM70-NEXT: setp.eq.b64 %p17, %rd13, 0;
+; SM70-NEXT: selp.f64 %rd34, %rd13, %rd1, %p17;
+; SM70-NEXT: setp.nan.f64 %p18, %rd13, %rd1;
+; SM70-NEXT: max.f64 %rd35, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
+; SM70-NEXT: setp.eq.f64 %p19, %rd13, %rd1;
+; SM70-NEXT: selp.f64 %rd37, %rd34, %rd36, %p19;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd13, %rd37;
+; SM70-NEXT: setp.ne.b64 %p20, %rd54, %rd13;
+; SM70-NEXT: @%p20 bra $L__BB287_9;
; SM70-NEXT: // %bb.10: // %atomicrmw.end37
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd63, [%rd17+40];
-; SM70-NEXT: setp.eq.b64 %p28, %rd2, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
; SM70-NEXT: $L__BB287_11: // %atomicrmw.start43
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b64 %rd14, %rd63;
-; SM70-NEXT: setp.nan.f64 %p26, %rd14, %rd2;
-; SM70-NEXT: max.f64 %rd43, %rd14, %rd2;
-; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
-; SM70-NEXT: setp.eq.b64 %p27, %rd14, 0;
-; SM70-NEXT: selp.f64 %rd45, %rd14, %rd44, %p27;
-; SM70-NEXT: selp.f64 %rd46, %rd2, %rd45, %p28;
-; SM70-NEXT: setp.eq.f64 %p29, %rd44, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd47, %rd46, %rd44, %p29;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd63, [%rd17+40], %rd14, %rd47;
-; SM70-NEXT: setp.ne.b64 %p30, %rd63, %rd14;
-; SM70-NEXT: @%p30 bra $L__BB287_11;
+; SM70-NEXT: mov.b64 %rd14, %rd55;
+; SM70-NEXT: setp.eq.b64 %p21, %rd14, 0;
+; SM70-NEXT: selp.f64 %rd38, %rd14, %rd2, %p21;
+; SM70-NEXT: setp.nan.f64 %p22, %rd14, %rd2;
+; SM70-NEXT: max.f64 %rd39, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
+; SM70-NEXT: setp.eq.f64 %p23, %rd14, %rd2;
+; SM70-NEXT: selp.f64 %rd41, %rd38, %rd40, %p23;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd14, %rd41;
+; SM70-NEXT: setp.ne.b64 %p24, %rd55, %rd14;
+; SM70-NEXT: @%p24 bra $L__BB287_11;
; SM70-NEXT: // %bb.12: // %atomicrmw.end42
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd64, [%rd17+48];
-; SM70-NEXT: setp.eq.b64 %p33, %rd11, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
; SM70-NEXT: $L__BB287_13: // %atomicrmw.start53
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p31, %rd64, %rd11;
-; SM70-NEXT: max.f64 %rd48, %rd64, %rd11;
-; SM70-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p31;
-; SM70-NEXT: setp.eq.b64 %p32, %rd64, 0;
-; SM70-NEXT: selp.f64 %rd50, %rd64, %rd49, %p32;
-; SM70-NEXT: selp.f64 %rd51, %rd11, %rd50, %p33;
-; SM70-NEXT: setp.eq.f64 %p34, %rd49, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd52, %rd51, %rd49, %p34;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd64, %rd52;
-; SM70-NEXT: setp.ne.b64 %p35, %rd15, %rd64;
-; SM70-NEXT: mov.b64 %rd64, %rd15;
-; SM70-NEXT: @%p35 bra $L__BB287_13;
+; SM70-NEXT: setp.eq.b64 %p25, %rd56, 0;
+; SM70-NEXT: selp.f64 %rd42, %rd56, %rd11, %p25;
+; SM70-NEXT: setp.nan.f64 %p26, %rd56, %rd11;
+; SM70-NEXT: max.f64 %rd43, %rd56, %rd11;
+; SM70-NEXT: selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
+; SM70-NEXT: setp.eq.f64 %p27, %rd56, %rd11;
+; SM70-NEXT: selp.f64 %rd45, %rd42, %rd44, %p27;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd15, [%rd17+48], %rd56, %rd45;
+; SM70-NEXT: setp.ne.b64 %p28, %rd15, %rd56;
+; SM70-NEXT: mov.b64 %rd56, %rd15;
+; SM70-NEXT: @%p28 bra $L__BB287_13;
; SM70-NEXT: // %bb.14: // %atomicrmw.end52
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd65, [%rd17+56];
-; SM70-NEXT: setp.eq.b64 %p38, %rd12, 0;
+; SM70-NEXT: ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
; SM70-NEXT: $L__BB287_15: // %atomicrmw.start58
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f64 %p36, %rd65, %rd12;
-; SM70-NEXT: max.f64 %rd53, %rd65, %rd12;
-; SM70-NEXT: selp.f64 %rd54, 0d7FF8000000000000, %rd53, %p36;
-; SM70-NEXT: setp.eq.b64 %p37, %rd65, 0;
-; SM70-NEXT: selp.f64 %rd55, %rd65, %rd54, %p37;
-; SM70-NEXT: selp.f64 %rd56, %rd12, %rd55, %p38;
-; SM70-NEXT: setp.eq.f64 %p39, %rd54, 0d0000000000000000;
-; SM70-NEXT: selp.f64 %rd57, %rd56, %rd54, %p39;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd65, %rd57;
-; SM70-NEXT: setp.ne.b64 %p40, %rd16, %rd65;
-; SM70-NEXT: mov.b64 %rd65, %rd16;
-; SM70-NEXT: @%p40 bra $L__BB287_15;
+; SM70-NEXT: setp.eq.b64 %p29, %rd57, 0;
+; SM70-NEXT: selp.f64 %rd46, %rd57, %rd12, %p29;
+; SM70-NEXT: setp.nan.f64 %p30, %rd57, %rd12;
+; SM70-NEXT: max.f64 %rd47, %rd57, %rd12;
+; SM70-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
+; SM70-NEXT: setp.eq.f64 %p31, %rd57, %rd12;
+; SM70-NEXT: selp.f64 %rd49, %rd46, %rd48, %p31;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
+; SM70-NEXT: setp.ne.b64 %p32, %rd16, %rd57;
+; SM70-NEXT: mov.b64 %rd57, %rd16;
+; SM70-NEXT: @%p32 bra $L__BB287_15;
; SM70-NEXT: // %bb.16: // %atomicrmw.end57
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd62, %rd63};
-; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
-; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
+; SM70-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM70-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -13527,8 +13407,8 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<7>;
; SM70-NEXT: .reg .b32 %r<15>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
@@ -13544,29 +13424,26 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NEXT: not.b32 %r2, %r7;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
; SM70-NEXT: $L__BB304_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r8, %r14, %r1;
; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, -32768;
; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: mov.b16 %rs7, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
; SM70-NEXT: shl.b32 %r10, %r9, %r1;
; SM70-NEXT: and.b32 %r11, %r14, %r2;
; SM70-NEXT: or.b32 %r12, %r11, %r10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r14;
; SM70-NEXT: mov.b32 %r14, %r3;
-; SM70-NEXT: @%p6 bra $L__BB304_1;
+; SM70-NEXT: @%p5 bra $L__BB304_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r13, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -13579,8 +13456,8 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<12>;
-; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<13>;
; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
@@ -13590,35 +13467,30 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
; SM70-NEXT: $L__BB305_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
-; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p11 bra $L__BB305_1;
+; SM70-NEXT: @%p9 bra $L__BB305_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13630,8 +13502,8 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<22>;
-; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<25>;
; SM70-NEXT: .reg .b32 %r<7>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
@@ -13643,55 +13515,46 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
-; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: $L__BB306_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
-; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
-; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
-; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
-; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r6;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: setp.lt.f16 %p10, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM70-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: setp.lt.f16 %p14, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM70-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r4, {%rs24, %rs20};
; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -13700,10 +13563,10 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB306_1;
+; SM70-NEXT: @%p17 bra $L__BB306_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -13715,8 +13578,8 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<43>;
-; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .pred %p<35>;
+; SM70-NEXT: .reg .b16 %rs<49>;
; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
@@ -13728,55 +13591,46 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM70-NEXT: setp.eq.b16 %p14, %rs17, -32768;
-; SM70-NEXT: setp.eq.b16 %p19, %rs16, -32768;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: $L__BB307_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM70-NEXT: setp.lt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.lt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.lt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r5, {%rs12, %rs8};
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
-; SM70-NEXT: setp.lt.f16 %p11, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs19, -32768;
-; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM70-NEXT: setp.lt.f16 %p16, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs18, -32768;
-; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r10;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: setp.lt.f16 %p10, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM70-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: setp.lt.f16 %p14, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM70-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r6, {%rs24, %rs20};
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -13785,64 +13639,55 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB307_1;
+; SM70-NEXT: @%p17 bra $L__BB307_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: $L__BB307_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
-; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
-; SM70-NEXT: setp.lt.f16 %p22, %rs33, %rs31;
-; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
-; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
-; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
-; SM70-NEXT: setp.eq.b16 %p24, %rs33, -32768;
-; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
-; SM70-NEXT: setp.eq.b16 %p25, %rs31, -32768;
-; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
-; SM70-NEXT: mov.b16 %rs38, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
-; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
-; SM70-NEXT: setp.lt.f16 %p27, %rs32, %rs30;
-; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
-; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
-; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
-; SM70-NEXT: setp.eq.b16 %p29, %rs32, -32768;
-; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
-; SM70-NEXT: setp.eq.b16 %p30, %rs30, -32768;
-; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
-; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
-; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
-; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r11;
+; SM70-NEXT: setp.eq.b16 %p18, %rs28, -32768;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM70-NEXT: setp.lt.f16 %p19, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM70-NEXT: setp.nan.f16 %p20, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs31, 0x7E00, %rs30, %p20;
+; SM70-NEXT: setp.eq.f16 %p21, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM70-NEXT: setp.eq.b16 %p22, %rs27, -32768;
+; SM70-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM70-NEXT: setp.lt.f16 %p23, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM70-NEXT: setp.nan.f16 %p24, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p24;
+; SM70-NEXT: setp.eq.f16 %p25, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM70-NEXT: mov.b32 %r7, {%rs36, %rs32};
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
-; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
-; SM70-NEXT: setp.lt.f16 %p32, %rs48, %rs46;
-; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
-; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
-; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
-; SM70-NEXT: setp.eq.b16 %p34, %rs48, -32768;
-; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
-; SM70-NEXT: setp.eq.b16 %p35, %rs46, -32768;
-; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
-; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
-; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
-; SM70-NEXT: setp.lt.f16 %p37, %rs47, %rs45;
-; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
-; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
-; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
-; SM70-NEXT: setp.eq.b16 %p39, %rs47, -32768;
-; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
-; SM70-NEXT: setp.eq.b16 %p40, %rs45, -32768;
-; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
-; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
-; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
-; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM70-NEXT: mov.b32 {%rs39, %rs40}, %r12;
+; SM70-NEXT: setp.eq.b16 %p26, %rs40, -32768;
+; SM70-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM70-NEXT: setp.lt.f16 %p27, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs43, 0x7E00, %rs42, %p28;
+; SM70-NEXT: setp.eq.f16 %p29, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs39, -32768;
+; SM70-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM70-NEXT: setp.lt.f16 %p31, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM70-NEXT: setp.nan.f16 %p32, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs47, 0x7E00, %rs46, %p32;
+; SM70-NEXT: setp.eq.f16 %p33, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM70-NEXT: mov.b32 %r8, {%rs48, %rs44};
; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
@@ -13851,10 +13696,10 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB307_3;
+; SM70-NEXT: @%p34 bra $L__BB307_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -13866,8 +13711,8 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<9>;
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<7>;
; SM70-NEXT: .reg .b32 %r<15>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
@@ -13883,29 +13728,26 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM70-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NEXT: not.b32 %r2, %r7;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
; SM70-NEXT: $L__BB308_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r8, %r14, %r1;
; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, 0;
; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: mov.b16 %rs7, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
; SM70-NEXT: shl.b32 %r10, %r9, %r1;
; SM70-NEXT: and.b32 %r11, %r14, %r2;
; SM70-NEXT: or.b32 %r12, %r11, %r10;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r14;
; SM70-NEXT: mov.b32 %r14, %r3;
-; SM70-NEXT: @%p6 bra $L__BB308_1;
+; SM70-NEXT: @%p5 bra $L__BB308_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r13, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -13918,8 +13760,8 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<12>;
-; SM70-NEXT: .reg .b16 %rs<16>;
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<13>;
; SM70-NEXT: .reg .b32 %r<5>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
@@ -13929,35 +13771,30 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
; SM70-NEXT: $L__BB309_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r4;
-; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NEXT: setp.ne.b32 %p11, %r1, %r4;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r4;
; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p11 bra $L__BB309_1;
+; SM70-NEXT: @%p9 bra $L__BB309_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -13969,8 +13806,8 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<22>;
-; SM70-NEXT: .reg .b16 %rs<30>;
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<25>;
; SM70-NEXT: .reg .b32 %r<7>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
@@ -13982,55 +13819,46 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
-; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: $L__BB310_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r5;
-; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r3, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r3, {%rs12, %rs8};
; SM70-NEXT: cvt.u64.u32 %rd3, %r3;
-; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r6;
-; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
-; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
-; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM70-NEXT: mov.b32 %r4, {%rs29, %rs24};
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r6;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: setp.gt.f16 %p10, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM70-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: setp.gt.f16 %p14, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM70-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r4, {%rs24, %rs20};
; SM70-NEXT: cvt.u64.u32 %rd4, %r4;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -14039,10 +13867,10 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB310_1;
+; SM70-NEXT: @%p17 bra $L__BB310_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -14054,8 +13882,8 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<43>;
-; SM70-NEXT: .reg .b16 %rs<59>;
+; SM70-NEXT: .reg .pred %p<35>;
+; SM70-NEXT: .reg .b16 %rs<49>;
; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
@@ -14067,55 +13895,46 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: mov.b32 {%rs16, %rs17}, %r2;
-; SM70-NEXT: setp.eq.b16 %p14, %rs17, 0;
-; SM70-NEXT: setp.eq.b16 %p19, %rs16, 0;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: $L__BB311_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r9;
-; SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs2;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs4, %rs2;
-; SM70-NEXT: selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: mov.b16 %rs9, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs6, %rs9;
-; SM70-NEXT: selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT: setp.gt.f16 %p2, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f16 %p3, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs7, 0x7E00, %rs6, %p3;
+; SM70-NEXT: setp.eq.f16 %p4, %rs4, %rs2;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NEXT: setp.gt.f16 %p6, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs11, %rs3, %rs1, %p6;
+; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NEXT: setp.nan.f16 %p7, %rs3, %rs1;
-; SM70-NEXT: selp.b16 %rs12, 0x7E00, %rs11, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs13, %rs3, %rs12, %p8;
-; SM70-NEXT: selp.b16 %rs14, %rs1, %rs13, %p9;
-; SM70-NEXT: setp.eq.f16 %p10, %rs12, %rs9;
-; SM70-NEXT: selp.b16 %rs15, %rs14, %rs12, %p10;
-; SM70-NEXT: mov.b32 %r5, {%rs15, %rs10};
+; SM70-NEXT: selp.b16 %rs11, 0x7E00, %rs10, %p7;
+; SM70-NEXT: setp.eq.f16 %p8, %rs3, %rs1;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r5, {%rs12, %rs8};
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: mov.b32 {%rs18, %rs19}, %r10;
-; SM70-NEXT: setp.gt.f16 %p11, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs20, %rs19, %rs17, %p11;
-; SM70-NEXT: setp.nan.f16 %p12, %rs19, %rs17;
-; SM70-NEXT: selp.b16 %rs21, 0x7E00, %rs20, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs19, 0;
-; SM70-NEXT: selp.b16 %rs22, %rs19, %rs21, %p13;
-; SM70-NEXT: selp.b16 %rs23, %rs17, %rs22, %p14;
-; SM70-NEXT: setp.eq.f16 %p15, %rs21, %rs9;
-; SM70-NEXT: selp.b16 %rs24, %rs23, %rs21, %p15;
-; SM70-NEXT: setp.gt.f16 %p16, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs25, %rs18, %rs16, %p16;
-; SM70-NEXT: setp.nan.f16 %p17, %rs18, %rs16;
-; SM70-NEXT: selp.b16 %rs26, 0x7E00, %rs25, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs18, 0;
-; SM70-NEXT: selp.b16 %rs27, %rs18, %rs26, %p18;
-; SM70-NEXT: selp.b16 %rs28, %rs16, %rs27, %p19;
-; SM70-NEXT: setp.eq.f16 %p20, %rs26, %rs9;
-; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p20;
-; SM70-NEXT: mov.b32 %r6, {%rs29, %rs24};
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r10;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: setp.gt.f16 %p10, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f16 %p11, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs19, 0x7E00, %rs18, %p11;
+; SM70-NEXT: setp.eq.f16 %p12, %rs16, %rs14;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: setp.gt.f16 %p14, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f16 %p15, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs23, 0x7E00, %rs22, %p15;
+; SM70-NEXT: setp.eq.f16 %p16, %rs15, %rs13;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r6, {%rs24, %rs20};
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
@@ -14124,64 +13943,55 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB311_1;
+; SM70-NEXT: @%p17 bra $L__BB311_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd12;
; SM70-NEXT: $L__BB311_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs30, %rs31}, %r3;
-; SM70-NEXT: mov.b32 {%rs32, %rs33}, %r11;
-; SM70-NEXT: setp.gt.f16 %p22, %rs33, %rs31;
-; SM70-NEXT: selp.b16 %rs34, %rs33, %rs31, %p22;
-; SM70-NEXT: setp.nan.f16 %p23, %rs33, %rs31;
-; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p23;
-; SM70-NEXT: setp.eq.b16 %p24, %rs33, 0;
-; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p24;
-; SM70-NEXT: setp.eq.b16 %p25, %rs31, 0;
-; SM70-NEXT: selp.b16 %rs37, %rs31, %rs36, %p25;
-; SM70-NEXT: mov.b16 %rs38, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p26, %rs35, %rs38;
-; SM70-NEXT: selp.b16 %rs39, %rs37, %rs35, %p26;
-; SM70-NEXT: setp.gt.f16 %p27, %rs32, %rs30;
-; SM70-NEXT: selp.b16 %rs40, %rs32, %rs30, %p27;
-; SM70-NEXT: setp.nan.f16 %p28, %rs32, %rs30;
-; SM70-NEXT: selp.b16 %rs41, 0x7E00, %rs40, %p28;
-; SM70-NEXT: setp.eq.b16 %p29, %rs32, 0;
-; SM70-NEXT: selp.b16 %rs42, %rs32, %rs41, %p29;
-; SM70-NEXT: setp.eq.b16 %p30, %rs30, 0;
-; SM70-NEXT: selp.b16 %rs43, %rs30, %rs42, %p30;
-; SM70-NEXT: setp.eq.f16 %p31, %rs41, %rs38;
-; SM70-NEXT: selp.b16 %rs44, %rs43, %rs41, %p31;
-; SM70-NEXT: mov.b32 %r7, {%rs44, %rs39};
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r11;
+; SM70-NEXT: setp.eq.b16 %p18, %rs28, 0;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM70-NEXT: setp.gt.f16 %p19, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM70-NEXT: setp.nan.f16 %p20, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs31, 0x7E00, %rs30, %p20;
+; SM70-NEXT: setp.eq.f16 %p21, %rs28, %rs26;
+; SM70-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM70-NEXT: setp.eq.b16 %p22, %rs27, 0;
+; SM70-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM70-NEXT: setp.gt.f16 %p23, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM70-NEXT: setp.nan.f16 %p24, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs35, 0x7E00, %rs34, %p24;
+; SM70-NEXT: setp.eq.f16 %p25, %rs27, %rs25;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM70-NEXT: mov.b32 %r7, {%rs36, %rs32};
; SM70-NEXT: cvt.u64.u32 %rd13, %r7;
-; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r4;
-; SM70-NEXT: mov.b32 {%rs47, %rs48}, %r12;
-; SM70-NEXT: setp.gt.f16 %p32, %rs48, %rs46;
-; SM70-NEXT: selp.b16 %rs49, %rs48, %rs46, %p32;
-; SM70-NEXT: setp.nan.f16 %p33, %rs48, %rs46;
-; SM70-NEXT: selp.b16 %rs50, 0x7E00, %rs49, %p33;
-; SM70-NEXT: setp.eq.b16 %p34, %rs48, 0;
-; SM70-NEXT: selp.b16 %rs51, %rs48, %rs50, %p34;
-; SM70-NEXT: setp.eq.b16 %p35, %rs46, 0;
-; SM70-NEXT: selp.b16 %rs52, %rs46, %rs51, %p35;
-; SM70-NEXT: setp.eq.f16 %p36, %rs50, %rs38;
-; SM70-NEXT: selp.b16 %rs53, %rs52, %rs50, %p36;
-; SM70-NEXT: setp.gt.f16 %p37, %rs47, %rs45;
-; SM70-NEXT: selp.b16 %rs54, %rs47, %rs45, %p37;
-; SM70-NEXT: setp.nan.f16 %p38, %rs47, %rs45;
-; SM70-NEXT: selp.b16 %rs55, 0x7E00, %rs54, %p38;
-; SM70-NEXT: setp.eq.b16 %p39, %rs47, 0;
-; SM70-NEXT: selp.b16 %rs56, %rs47, %rs55, %p39;
-; SM70-NEXT: setp.eq.b16 %p40, %rs45, 0;
-; SM70-NEXT: selp.b16 %rs57, %rs45, %rs56, %p40;
-; SM70-NEXT: setp.eq.f16 %p41, %rs55, %rs38;
-; SM70-NEXT: selp.b16 %rs58, %rs57, %rs55, %p41;
-; SM70-NEXT: mov.b32 %r8, {%rs58, %rs53};
+; SM70-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM70-NEXT: mov.b32 {%rs39, %rs40}, %r12;
+; SM70-NEXT: setp.eq.b16 %p26, %rs40, 0;
+; SM70-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM70-NEXT: setp.gt.f16 %p27, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM70-NEXT: setp.nan.f16 %p28, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs43, 0x7E00, %rs42, %p28;
+; SM70-NEXT: setp.eq.f16 %p29, %rs40, %rs38;
+; SM70-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs39, 0;
+; SM70-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM70-NEXT: setp.gt.f16 %p31, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM70-NEXT: setp.nan.f16 %p32, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs47, 0x7E00, %rs46, %p32;
+; SM70-NEXT: setp.eq.f16 %p33, %rs39, %rs37;
+; SM70-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM70-NEXT: mov.b32 %r8, {%rs48, %rs44};
; SM70-NEXT: cvt.u64.u32 %rd14, %r8;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
@@ -14190,10 +14000,10 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
+; SM70-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB311_3;
+; SM70-NEXT: @%p34 bra $L__BB311_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -15593,9 +15403,9 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<8>;
-; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<18>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -15609,38 +15419,34 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 %r6, 65535;
; SM70-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
; SM70-NEXT: $L__BB328_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, -32768;
; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM70-NEXT: shl.b32 %r15, %r14, %r1;
-; SM70-NEXT: and.b32 %r16, %r19, %r2;
-; SM70-NEXT: or.b32 %r17, %r16, %r15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM70-NEXT: mov.b32 %r19, %r3;
-; SM70-NEXT: @%p6 bra $L__BB328_1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p5 bra $L__BB328_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -15649,57 +15455,49 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<12>;
-; SM70-NEXT: .reg .b16 %rs<15>;
-; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: $L__BB329_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: setp.lt.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r10, %r8;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
-; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
-; SM70-NEXT: mov.b32 %r16, %r1;
-; SM70-NEXT: @%p11 bra $L__BB329_1;
+; SM70-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r12;
+; SM70-NEXT: mov.b32 %r12, %r1;
+; SM70-NEXT: @%p9 bra $L__BB329_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -15711,9 +15509,9 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<22>;
-; SM70-NEXT: .reg .b16 %rs<29>;
-; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<23>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -15721,96 +15519,80 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs14;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
; SM70-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
-; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
-; SM70-NEXT: shl.b32 %r23, %r22, 16;
-; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
; SM70-NEXT: $L__BB330_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.lt.f32 %p1, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
+; SM70-NEXT: setp.lt.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.lt.f32 %p6, %r10, %r8;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
-; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
-; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r22;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs16;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: setp.lt.f32 %p10, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f32 %p11, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM70-NEXT: setp.eq.f32 %p12, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs15;
; SM70-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NEXT: setp.lt.f32 %p11, %r19, %r17;
-; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
-; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
-; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
-; SM70-NEXT: shl.b32 %r21, %r20, 16;
-; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
-; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
-; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: setp.lt.f32 %p16, %r25, %r23;
-; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
-; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
-; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
-; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
-; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: setp.lt.f32 %p14, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f32 %p15, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM70-NEXT: setp.eq.f32 %p16, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r20, {%rs24, %rs20};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r20;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB330_1;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB330_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -15819,9 +15601,9 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<43>;
-; SM70-NEXT: .reg .b16 %rs<57>;
-; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .pred %p<35>;
+; SM70-NEXT: .reg .b16 %rs<49>;
+; SM70-NEXT: .reg .b32 %r<45>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -15829,185 +15611,153 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, -32768;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, -32768;
-; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
-; SM70-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NEXT: setp.eq.b16 %p14, %rs16, -32768;
-; SM70-NEXT: setp.eq.b16 %p19, %rs15, -32768;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: $L__BB331_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r41;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.lt.f32 %p1, %r8, %r6;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, -32768;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: setp.lt.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.lt.f32 %p6, %r14, %r12;
+; SM70-NEXT: setp.lt.f32 %p6, %r12, %r10;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, -32768;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
-; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: setp.eq.f32 %p8, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r13, {%rs12, %rs8};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r42;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, -32768;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.lt.f32 %p10, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f32 %p11, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM70-NEXT: setp.eq.f32 %p12, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, -32768;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs13;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
; SM70-NEXT: shl.b32 %r21, %r20, 16;
-; SM70-NEXT: setp.lt.f32 %p11, %r21, %r19;
-; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
-; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs18, -32768;
-; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
-; SM70-NEXT: shl.b32 %r23, %r22, 16;
-; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
-; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
-; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: setp.lt.f32 %p16, %r27, %r25;
-; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
-; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs17, -32768;
-; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
-; SM70-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
-; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
-; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: setp.lt.f32 %p14, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f32 %p15, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM70-NEXT: setp.eq.f32 %p16, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r22, {%rs24, %rs20};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r22;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r42;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB331_1;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB331_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd12;
; SM70-NEXT: $L__BB331_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
-; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
-; SM70-NEXT: shl.b32 %r32, %r31, 16;
-; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
-; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
-; SM70-NEXT: shl.b32 %r34, %r33, 16;
-; SM70-NEXT: setp.lt.f32 %p22, %r34, %r32;
-; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
-; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
-; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
-; SM70-NEXT: setp.eq.b16 %p24, %rs32, -32768;
-; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
-; SM70-NEXT: setp.eq.b16 %p25, %rs30, -32768;
-; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
-; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
-; SM70-NEXT: shl.b32 %r36, %r35, 16;
-; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
-; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
-; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
-; SM70-NEXT: shl.b32 %r38, %r37, 16;
-; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
-; SM70-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-NEXT: setp.lt.f32 %p27, %r40, %r38;
-; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
-; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
-; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
-; SM70-NEXT: setp.eq.b16 %p29, %rs31, -32768;
-; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
-; SM70-NEXT: setp.eq.b16 %p30, %rs29, -32768;
-; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
-; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
-; SM70-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
-; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
-; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
-; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
-; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
-; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
-; SM70-NEXT: shl.b32 %r45, %r44, 16;
-; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
-; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
-; SM70-NEXT: shl.b32 %r47, %r46, 16;
-; SM70-NEXT: setp.lt.f32 %p32, %r47, %r45;
-; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
-; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
-; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
-; SM70-NEXT: setp.eq.b16 %p34, %rs46, -32768;
-; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
-; SM70-NEXT: setp.eq.b16 %p35, %rs44, -32768;
-; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
-; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
-; SM70-NEXT: shl.b32 %r49, %r48, 16;
-; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
-; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
-; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
-; SM70-NEXT: shl.b32 %r51, %r50, 16;
-; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
-; SM70-NEXT: shl.b32 %r53, %r52, 16;
-; SM70-NEXT: setp.lt.f32 %p37, %r53, %r51;
-; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
-; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
-; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
-; SM70-NEXT: setp.eq.b16 %p39, %rs45, -32768;
-; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
-; SM70-NEXT: setp.eq.b16 %p40, %rs43, -32768;
-; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
-; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
-; SM70-NEXT: shl.b32 %r55, %r54, 16;
-; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
-; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
-; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
-; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r43;
+; SM70-NEXT: setp.eq.b16 %p18, %rs28, -32768;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs26;
+; SM70-NEXT: shl.b32 %r24, %r23, 16;
+; SM70-NEXT: cvt.u32.u16 %r25, %rs28;
+; SM70-NEXT: shl.b32 %r26, %r25, 16;
+; SM70-NEXT: setp.lt.f32 %p19, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM70-NEXT: setp.nan.f32 %p20, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs31, 0x7FC0, %rs30, %p20;
+; SM70-NEXT: setp.eq.f32 %p21, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM70-NEXT: setp.eq.b16 %p22, %rs27, -32768;
+; SM70-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs25;
+; SM70-NEXT: shl.b32 %r28, %r27, 16;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs27;
+; SM70-NEXT: shl.b32 %r30, %r29, 16;
+; SM70-NEXT: setp.lt.f32 %p23, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM70-NEXT: setp.nan.f32 %p24, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs35, 0x7FC0, %rs34, %p24;
+; SM70-NEXT: setp.eq.f32 %p25, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM70-NEXT: mov.b32 %r31, {%rs36, %rs32};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r31;
+; SM70-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM70-NEXT: mov.b32 {%rs39, %rs40}, %r44;
+; SM70-NEXT: setp.eq.b16 %p26, %rs40, -32768;
+; SM70-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM70-NEXT: cvt.u32.u16 %r32, %rs38;
+; SM70-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs40;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: setp.lt.f32 %p27, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs43, 0x7FC0, %rs42, %p28;
+; SM70-NEXT: setp.eq.f32 %p29, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs39, -32768;
+; SM70-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs37;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs39;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: setp.lt.f32 %p31, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM70-NEXT: setp.nan.f32 %p32, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs47, 0x7FC0, %rs46, %p32;
+; SM70-NEXT: setp.eq.f32 %p33, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM70-NEXT: mov.b32 %r40, {%rs48, %rs44};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r40;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r44;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB331_3;
+; SM70-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM70-NEXT: @%p34 bra $L__BB331_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -16016,9 +15766,9 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<8>;
-; SM70-NEXT: .reg .b32 %r<20>;
+; SM70-NEXT: .reg .pred %p<6>;
+; SM70-NEXT: .reg .b16 %rs<7>;
+; SM70-NEXT: .reg .b32 %r<18>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -16032,38 +15782,34 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 %r6, 65535;
; SM70-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
; SM70-NEXT: $L__BB332_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NEXT: shr.u32 %r8, %r17, %r1;
; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM70-NEXT: setp.eq.b16 %p1, %rs2, 0;
; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM70-NEXT: shl.b32 %r15, %r14, %r1;
-; SM70-NEXT: and.b32 %r16, %r19, %r2;
-; SM70-NEXT: or.b32 %r17, %r16, %r15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM70-NEXT: mov.b32 %r19, %r3;
-; SM70-NEXT: @%p6 bra $L__BB332_1;
+; SM70-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NEXT: mov.b32 %r17, %r3;
+; SM70-NEXT: @%p5 bra $L__BB332_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NEXT: shr.u32 %r16, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NEXT: st.param.b16 [func_retval0], %r16;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -16072,57 +15818,49 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<12>;
-; SM70-NEXT: .reg .b16 %rs<15>;
-; SM70-NEXT: .reg .b32 %r<17>;
+; SM70-NEXT: .reg .pred %p<10>;
+; SM70-NEXT: .reg .b16 %rs<13>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: $L__BB333_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r16;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r12;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: setp.gt.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r10, %r8;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r16, %r15;
-; SM70-NEXT: setp.ne.b32 %p11, %r1, %r16;
-; SM70-NEXT: mov.b32 %r16, %r1;
-; SM70-NEXT: @%p11 bra $L__BB333_1;
+; SM70-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
+; SM70-NEXT: setp.ne.b32 %p9, %r1, %r12;
+; SM70-NEXT: mov.b32 %r12, %r1;
+; SM70-NEXT: @%p9 bra $L__BB333_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -16134,9 +15872,9 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<22>;
-; SM70-NEXT: .reg .b16 %rs<29>;
-; SM70-NEXT: .reg .b32 %r<31>;
+; SM70-NEXT: .reg .pred %p<18>;
+; SM70-NEXT: .reg .b16 %rs<25>;
+; SM70-NEXT: .reg .b32 %r<23>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -16144,96 +15882,80 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r29, %rd2;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
-; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs14;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
; SM70-NEXT: shl.b32 %r17, %r16, 16;
-; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
-; SM70-NEXT: cvt.u32.u16 %r22, %rs15;
-; SM70-NEXT: shl.b32 %r23, %r22, 16;
-; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
; SM70-NEXT: $L__BB334_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r29;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r6, %r4;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r8, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
+; SM70-NEXT: setp.gt.f32 %p2, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r6, %r4;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NEXT: shl.b32 %r10, %r9, 16;
+; SM70-NEXT: setp.gt.f32 %p6, %r10, %r8;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT: setp.nan.f32 %p7, %r10, %r8;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r14, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT: cvt.u64.u32 %rd3, %r15;
-; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r30;
-; SM70-NEXT: cvt.u32.u16 %r18, %rs18;
+; SM70-NEXT: setp.eq.f32 %p8, %r10, %r8;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r11, {%rs12, %rs8};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r22;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs16;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: setp.gt.f32 %p10, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f32 %p11, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM70-NEXT: setp.eq.f32 %p12, %r15, %r13;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs15;
; SM70-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NEXT: setp.gt.f32 %p11, %r19, %r17;
-; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM70-NEXT: setp.nan.f32 %p12, %r19, %r17;
-; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
-; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
-; SM70-NEXT: shl.b32 %r21, %r20, 16;
-; SM70-NEXT: setp.eq.f32 %p15, %r21, 0f00000000;
-; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM70-NEXT: cvt.u32.u16 %r24, %rs17;
-; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: setp.gt.f32 %p16, %r25, %r23;
-; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM70-NEXT: setp.nan.f32 %p17, %r25, %r23;
-; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
-; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs25;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: setp.eq.f32 %p20, %r27, 0f00000000;
-; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM70-NEXT: mov.b32 %r28, {%rs28, %rs23};
-; SM70-NEXT: cvt.u64.u32 %rd4, %r28;
+; SM70-NEXT: setp.gt.f32 %p14, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f32 %p15, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM70-NEXT: setp.eq.f32 %p16, %r19, %r17;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r20, {%rs24, %rs20};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r20;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r29;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r30;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r21;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r22;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r29, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB334_1;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r21, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB334_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r29, %r30};
+; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -16242,9 +15964,9 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM70: {
-; SM70-NEXT: .reg .pred %p<43>;
-; SM70-NEXT: .reg .b16 %rs<57>;
-; SM70-NEXT: .reg .b32 %r<61>;
+; SM70-NEXT: .reg .pred %p<35>;
+; SM70-NEXT: .reg .b16 %rs<49>;
+; SM70-NEXT: .reg .b32 %r<45>;
; SM70-NEXT: .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
@@ -16252,185 +15974,153 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r57, %rd2;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; SM70-NEXT: shl.b32 %r6, %r5, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs2, 0;
-; SM70-NEXT: setp.eq.b16 %p9, %rs1, 0;
-; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r18, %rs16;
-; SM70-NEXT: shl.b32 %r19, %r18, 16;
-; SM70-NEXT: setp.eq.b16 %p14, %rs16, 0;
-; SM70-NEXT: setp.eq.b16 %p19, %rs15, 0;
+; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: $L__BB335_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r57;
+; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r41;
+; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
+; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r8, %r6;
-; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r8, %r6;
-; SM70-NEXT: selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs4, 0;
-; SM70-NEXT: selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT: selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT: setp.gt.f32 %p2, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs6, %rs4, %rs2, %p2;
+; SM70-NEXT: setp.nan.f32 %p3, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs7, 0x7FC0, %rs6, %p3;
+; SM70-NEXT: setp.eq.f32 %p4, %r8, %r6;
+; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
+; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
+; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r9, %rs1;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r10, 0f00000000;
-; SM70-NEXT: selp.b16 %rs9, %rs8, %rs6, %p5;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs1;
+; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs3;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: setp.gt.f32 %p6, %r14, %r12;
+; SM70-NEXT: setp.gt.f32 %p6, %r12, %r10;
; SM70-NEXT: selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT: setp.nan.f32 %p7, %r14, %r12;
+; SM70-NEXT: setp.nan.f32 %p7, %r12, %r10;
; SM70-NEXT: selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT: setp.eq.b16 %p8, %rs3, 0;
-; SM70-NEXT: selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT: selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs11;
-; SM70-NEXT: shl.b32 %r16, %r15, 16;
-; SM70-NEXT: setp.eq.f32 %p10, %r16, 0f00000000;
-; SM70-NEXT: selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT: mov.b32 %r17, {%rs14, %rs9};
-; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM70-NEXT: mov.b32 {%rs17, %rs18}, %r58;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs18;
+; SM70-NEXT: setp.eq.f32 %p8, %r12, %r10;
+; SM70-NEXT: selp.b16 %rs12, %rs9, %rs11, %p8;
+; SM70-NEXT: mov.b32 %r13, {%rs12, %rs8};
+; SM70-NEXT: cvt.u64.u32 %rd3, %r13;
+; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r42;
+; SM70-NEXT: setp.eq.b16 %p9, %rs16, 0;
+; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
+; SM70-NEXT: setp.gt.f32 %p10, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs18, %rs16, %rs14, %p10;
+; SM70-NEXT: setp.nan.f32 %p11, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs19, 0x7FC0, %rs18, %p11;
+; SM70-NEXT: setp.eq.f32 %p12, %r17, %r15;
+; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
+; SM70-NEXT: setp.eq.b16 %p13, %rs15, 0;
+; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r18, %rs13;
+; SM70-NEXT: shl.b32 %r19, %r18, 16;
+; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
; SM70-NEXT: shl.b32 %r21, %r20, 16;
-; SM70-NEXT: setp.gt.f32 %p11, %r21, %r19;
-; SM70-NEXT: selp.b16 %rs19, %rs18, %rs16, %p11;
-; SM70-NEXT: setp.nan.f32 %p12, %r21, %r19;
-; SM70-NEXT: selp.b16 %rs20, 0x7FC0, %rs19, %p12;
-; SM70-NEXT: setp.eq.b16 %p13, %rs18, 0;
-; SM70-NEXT: selp.b16 %rs21, %rs18, %rs20, %p13;
-; SM70-NEXT: selp.b16 %rs22, %rs16, %rs21, %p14;
-; SM70-NEXT: cvt.u32.u16 %r22, %rs20;
-; SM70-NEXT: shl.b32 %r23, %r22, 16;
-; SM70-NEXT: setp.eq.f32 %p15, %r23, 0f00000000;
-; SM70-NEXT: selp.b16 %rs23, %rs22, %rs20, %p15;
-; SM70-NEXT: cvt.u32.u16 %r24, %rs15;
-; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs17;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
-; SM70-NEXT: setp.gt.f32 %p16, %r27, %r25;
-; SM70-NEXT: selp.b16 %rs24, %rs17, %rs15, %p16;
-; SM70-NEXT: setp.nan.f32 %p17, %r27, %r25;
-; SM70-NEXT: selp.b16 %rs25, 0x7FC0, %rs24, %p17;
-; SM70-NEXT: setp.eq.b16 %p18, %rs17, 0;
-; SM70-NEXT: selp.b16 %rs26, %rs17, %rs25, %p18;
-; SM70-NEXT: selp.b16 %rs27, %rs15, %rs26, %p19;
-; SM70-NEXT: cvt.u32.u16 %r28, %rs25;
-; SM70-NEXT: shl.b32 %r29, %r28, 16;
-; SM70-NEXT: setp.eq.f32 %p20, %r29, 0f00000000;
-; SM70-NEXT: selp.b16 %rs28, %rs27, %rs25, %p20;
-; SM70-NEXT: mov.b32 %r30, {%rs28, %rs23};
-; SM70-NEXT: cvt.u64.u32 %rd4, %r30;
+; SM70-NEXT: setp.gt.f32 %p14, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs22, %rs15, %rs13, %p14;
+; SM70-NEXT: setp.nan.f32 %p15, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs23, 0x7FC0, %rs22, %p15;
+; SM70-NEXT: setp.eq.f32 %p16, %r21, %r19;
+; SM70-NEXT: selp.b16 %rs24, %rs21, %rs23, %p16;
+; SM70-NEXT: mov.b32 %r22, {%rs24, %rs20};
+; SM70-NEXT: cvt.u64.u32 %rd4, %r22;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r57;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r58;
+; SM70-NEXT: cvt.u64.u32 %rd7, %r41;
+; SM70-NEXT: cvt.u64.u32 %rd8, %r42;
; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p21, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r58}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r57, %rd11;
-; SM70-NEXT: @%p21 bra $L__BB335_1;
+; SM70-NEXT: setp.ne.b64 %p17, %rd11, %rd10;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
+; SM70-NEXT: cvt.u32.u64 %r41, %rd11;
+; SM70-NEXT: @%p17 bra $L__BB335_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r59, %rd12;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd12;
; SM70-NEXT: $L__BB335_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: mov.b32 {%rs29, %rs30}, %r3;
-; SM70-NEXT: cvt.u32.u16 %r31, %rs30;
-; SM70-NEXT: shl.b32 %r32, %r31, 16;
-; SM70-NEXT: mov.b32 {%rs31, %rs32}, %r59;
-; SM70-NEXT: cvt.u32.u16 %r33, %rs32;
-; SM70-NEXT: shl.b32 %r34, %r33, 16;
-; SM70-NEXT: setp.gt.f32 %p22, %r34, %r32;
-; SM70-NEXT: selp.b16 %rs33, %rs32, %rs30, %p22;
-; SM70-NEXT: setp.nan.f32 %p23, %r34, %r32;
-; SM70-NEXT: selp.b16 %rs34, 0x7FC0, %rs33, %p23;
-; SM70-NEXT: setp.eq.b16 %p24, %rs32, 0;
-; SM70-NEXT: selp.b16 %rs35, %rs32, %rs34, %p24;
-; SM70-NEXT: setp.eq.b16 %p25, %rs30, 0;
-; SM70-NEXT: selp.b16 %rs36, %rs30, %rs35, %p25;
-; SM70-NEXT: cvt.u32.u16 %r35, %rs34;
-; SM70-NEXT: shl.b32 %r36, %r35, 16;
-; SM70-NEXT: setp.eq.f32 %p26, %r36, 0f00000000;
-; SM70-NEXT: selp.b16 %rs37, %rs36, %rs34, %p26;
-; SM70-NEXT: cvt.u32.u16 %r37, %rs29;
-; SM70-NEXT: shl.b32 %r38, %r37, 16;
-; SM70-NEXT: cvt.u32.u16 %r39, %rs31;
-; SM70-NEXT: shl.b32 %r40, %r39, 16;
-; SM70-NEXT: setp.gt.f32 %p27, %r40, %r38;
-; SM70-NEXT: selp.b16 %rs38, %rs31, %rs29, %p27;
-; SM70-NEXT: setp.nan.f32 %p28, %r40, %r38;
-; SM70-NEXT: selp.b16 %rs39, 0x7FC0, %rs38, %p28;
-; SM70-NEXT: setp.eq.b16 %p29, %rs31, 0;
-; SM70-NEXT: selp.b16 %rs40, %rs31, %rs39, %p29;
-; SM70-NEXT: setp.eq.b16 %p30, %rs29, 0;
-; SM70-NEXT: selp.b16 %rs41, %rs29, %rs40, %p30;
-; SM70-NEXT: cvt.u32.u16 %r41, %rs39;
-; SM70-NEXT: shl.b32 %r42, %r41, 16;
-; SM70-NEXT: setp.eq.f32 %p31, %r42, 0f00000000;
-; SM70-NEXT: selp.b16 %rs42, %rs41, %rs39, %p31;
-; SM70-NEXT: mov.b32 %r43, {%rs42, %rs37};
-; SM70-NEXT: cvt.u64.u32 %rd13, %r43;
-; SM70-NEXT: mov.b32 {%rs43, %rs44}, %r4;
-; SM70-NEXT: cvt.u32.u16 %r44, %rs44;
-; SM70-NEXT: shl.b32 %r45, %r44, 16;
-; SM70-NEXT: mov.b32 {%rs45, %rs46}, %r60;
-; SM70-NEXT: cvt.u32.u16 %r46, %rs46;
-; SM70-NEXT: shl.b32 %r47, %r46, 16;
-; SM70-NEXT: setp.gt.f32 %p32, %r47, %r45;
-; SM70-NEXT: selp.b16 %rs47, %rs46, %rs44, %p32;
-; SM70-NEXT: setp.nan.f32 %p33, %r47, %r45;
-; SM70-NEXT: selp.b16 %rs48, 0x7FC0, %rs47, %p33;
-; SM70-NEXT: setp.eq.b16 %p34, %rs46, 0;
-; SM70-NEXT: selp.b16 %rs49, %rs46, %rs48, %p34;
-; SM70-NEXT: setp.eq.b16 %p35, %rs44, 0;
-; SM70-NEXT: selp.b16 %rs50, %rs44, %rs49, %p35;
-; SM70-NEXT: cvt.u32.u16 %r48, %rs48;
-; SM70-NEXT: shl.b32 %r49, %r48, 16;
-; SM70-NEXT: setp.eq.f32 %p36, %r49, 0f00000000;
-; SM70-NEXT: selp.b16 %rs51, %rs50, %rs48, %p36;
-; SM70-NEXT: cvt.u32.u16 %r50, %rs43;
-; SM70-NEXT: shl.b32 %r51, %r50, 16;
-; SM70-NEXT: cvt.u32.u16 %r52, %rs45;
-; SM70-NEXT: shl.b32 %r53, %r52, 16;
-; SM70-NEXT: setp.gt.f32 %p37, %r53, %r51;
-; SM70-NEXT: selp.b16 %rs52, %rs45, %rs43, %p37;
-; SM70-NEXT: setp.nan.f32 %p38, %r53, %r51;
-; SM70-NEXT: selp.b16 %rs53, 0x7FC0, %rs52, %p38;
-; SM70-NEXT: setp.eq.b16 %p39, %rs45, 0;
-; SM70-NEXT: selp.b16 %rs54, %rs45, %rs53, %p39;
-; SM70-NEXT: setp.eq.b16 %p40, %rs43, 0;
-; SM70-NEXT: selp.b16 %rs55, %rs43, %rs54, %p40;
-; SM70-NEXT: cvt.u32.u16 %r54, %rs53;
-; SM70-NEXT: shl.b32 %r55, %r54, 16;
-; SM70-NEXT: setp.eq.f32 %p41, %r55, 0f00000000;
-; SM70-NEXT: selp.b16 %rs56, %rs55, %rs53, %p41;
-; SM70-NEXT: mov.b32 %r56, {%rs56, %rs51};
-; SM70-NEXT: cvt.u64.u32 %rd14, %r56;
+; SM70-NEXT: mov.b32 {%rs25, %rs26}, %r3;
+; SM70-NEXT: mov.b32 {%rs27, %rs28}, %r43;
+; SM70-NEXT: setp.eq.b16 %p18, %rs28, 0;
+; SM70-NEXT: selp.b16 %rs29, %rs28, %rs26, %p18;
+; SM70-NEXT: cvt.u32.u16 %r23, %rs26;
+; SM70-NEXT: shl.b32 %r24, %r23, 16;
+; SM70-NEXT: cvt.u32.u16 %r25, %rs28;
+; SM70-NEXT: shl.b32 %r26, %r25, 16;
+; SM70-NEXT: setp.gt.f32 %p19, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs30, %rs28, %rs26, %p19;
+; SM70-NEXT: setp.nan.f32 %p20, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs31, 0x7FC0, %rs30, %p20;
+; SM70-NEXT: setp.eq.f32 %p21, %r26, %r24;
+; SM70-NEXT: selp.b16 %rs32, %rs29, %rs31, %p21;
+; SM70-NEXT: setp.eq.b16 %p22, %rs27, 0;
+; SM70-NEXT: selp.b16 %rs33, %rs27, %rs25, %p22;
+; SM70-NEXT: cvt.u32.u16 %r27, %rs25;
+; SM70-NEXT: shl.b32 %r28, %r27, 16;
+; SM70-NEXT: cvt.u32.u16 %r29, %rs27;
+; SM70-NEXT: shl.b32 %r30, %r29, 16;
+; SM70-NEXT: setp.gt.f32 %p23, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs34, %rs27, %rs25, %p23;
+; SM70-NEXT: setp.nan.f32 %p24, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs35, 0x7FC0, %rs34, %p24;
+; SM70-NEXT: setp.eq.f32 %p25, %r30, %r28;
+; SM70-NEXT: selp.b16 %rs36, %rs33, %rs35, %p25;
+; SM70-NEXT: mov.b32 %r31, {%rs36, %rs32};
+; SM70-NEXT: cvt.u64.u32 %rd13, %r31;
+; SM70-NEXT: mov.b32 {%rs37, %rs38}, %r4;
+; SM70-NEXT: mov.b32 {%rs39, %rs40}, %r44;
+; SM70-NEXT: setp.eq.b16 %p26, %rs40, 0;
+; SM70-NEXT: selp.b16 %rs41, %rs40, %rs38, %p26;
+; SM70-NEXT: cvt.u32.u16 %r32, %rs38;
+; SM70-NEXT: shl.b32 %r33, %r32, 16;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs40;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
+; SM70-NEXT: setp.gt.f32 %p27, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs42, %rs40, %rs38, %p27;
+; SM70-NEXT: setp.nan.f32 %p28, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs43, 0x7FC0, %rs42, %p28;
+; SM70-NEXT: setp.eq.f32 %p29, %r35, %r33;
+; SM70-NEXT: selp.b16 %rs44, %rs41, %rs43, %p29;
+; SM70-NEXT: setp.eq.b16 %p30, %rs39, 0;
+; SM70-NEXT: selp.b16 %rs45, %rs39, %rs37, %p30;
+; SM70-NEXT: cvt.u32.u16 %r36, %rs37;
+; SM70-NEXT: shl.b32 %r37, %r36, 16;
+; SM70-NEXT: cvt.u32.u16 %r38, %rs39;
+; SM70-NEXT: shl.b32 %r39, %r38, 16;
+; SM70-NEXT: setp.gt.f32 %p31, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs46, %rs39, %rs37, %p31;
+; SM70-NEXT: setp.nan.f32 %p32, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs47, 0x7FC0, %rs46, %p32;
+; SM70-NEXT: setp.eq.f32 %p33, %r39, %r37;
+; SM70-NEXT: selp.b16 %rs48, %rs45, %rs47, %p33;
+; SM70-NEXT: mov.b32 %r40, {%rs48, %rs44};
+; SM70-NEXT: cvt.u64.u32 %rd14, %r40;
; SM70-NEXT: shl.b64 %rd15, %rd14, 32;
; SM70-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM70-NEXT: cvt.u64.u32 %rd17, %r59;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r60;
+; SM70-NEXT: cvt.u64.u32 %rd17, %r43;
+; SM70-NEXT: cvt.u64.u32 %rd18, %r44;
; SM70-NEXT: shl.b64 %rd19, %rd18, 32;
; SM70-NEXT: or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
-; SM70-NEXT: setp.ne.b64 %p42, %rd21, %rd20;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r60}, %rd21; }
-; SM70-NEXT: cvt.u32.u64 %r59, %rd21;
-; SM70-NEXT: @%p42 bra $L__BB335_3;
+; SM70-NEXT: setp.ne.b64 %p34, %rd21, %rd20;
+; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
+; SM70-NEXT: cvt.u32.u64 %r43, %rd21;
+; SM70-NEXT: @%p34 bra $L__BB335_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r57, %r58, %r59, %r60};
+; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index 2cf9406bd0447..a7ea56b5719aa 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -10234,88 +10234,88 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
ret <8 x i64> %retval
}
-define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
-; SM90-LABEL: fadd_acq_rel_v1f32_global_cta(
+define <1 x float> @fadd_acq_rel_v1f32_shared_cta(ptr addrspace(3) %addr, <1 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v1f32_shared_cta(
; SM90: {
; SM90-NEXT: .reg .b32 %r<3>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_shared_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
}
-define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
-; SM90-LABEL: fadd_acq_rel_v2f32_global_cta(
+define <2 x float> @fadd_acq_rel_v2f32_shared_cta(ptr addrspace(3) %addr, <2 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v2f32_shared_cta(
; SM90: {
; SM90-NEXT: .reg .b32 %r<5>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
+; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_shared_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r3, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
}
-define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
-; SM90-LABEL: fadd_acq_rel_v4f32_global_cta(
+define <4 x float> @fadd_acq_rel_v4f32_shared_cta(ptr addrspace(3) %addr, <4 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v4f32_shared_cta(
; SM90: {
; SM90-NEXT: .reg .b32 %r<9>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_shared_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r5, [%rd1], %r1;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r6, [%rd1+4], %r2;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r7, [%rd1+8], %r3;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
}
-define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
-; SM90-LABEL: fadd_acq_rel_v8f32_global_cta(
+define <8 x float> @fadd_acq_rel_v8f32_shared_cta(ptr addrspace(3) %addr, <8 x float> %val) {
+; SM90-LABEL: fadd_acq_rel_v8f32_shared_cta(
; SM90: {
; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
+; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_shared_cta_param_1+16];
+; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_shared_cta_param_1];
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r9, [%rd1], %r5;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r10, [%rd1+4], %r6;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r11, [%rd1+8], %r7;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r12, [%rd1+12], %r8;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r13, [%rd1+16], %r1;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r14, [%rd1+20], %r2;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r15, [%rd1+24], %r3;
+; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
- %retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
+ %retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
}
@@ -12516,29 +12516,27 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM90-LABEL: fminimum_acq_rel_v1f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<6>;
-; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
-; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM90-NEXT: min.f64 %rd4, %rd9, %rd1;
-; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM90-NEXT: mov.b64 %rd9, %rd2;
-; SM90-NEXT: @%p5 bra $L__BB280_1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd8, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM90-NEXT: min.f64 %rd5, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM90-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM90-NEXT: mov.b64 %rd8, %rd2;
+; SM90-NEXT: @%p4 bra $L__BB280_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12550,8 +12548,8 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<10>;
-; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-NEXT: .reg .pred %p<8>;
+; SM90-NEXT: .reg .b64 %rd<20>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fminimum_acq_rel_v2f64_global_cta_param_1];
@@ -12563,42 +12561,38 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: mov.b128 {%rd18, %rd19}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd4, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p7, %rd5, -9223372036854775808;
; SM90-NEXT: $L__BB281_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
-; SM90-NEXT: min.f64 %rd7, %rd20, %rd4;
-; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd20, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
-; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
-; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
-; SM90-NEXT: min.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: setp.eq.b64 %p1, %rd18, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd7, %rd18, %rd4, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd18, %rd4;
+; SM90-NEXT: min.f64 %rd8, %rd18, %rd4;
+; SM90-NEXT: selp.f64 %rd9, 0d7FF8000000000000, %rd8, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd18, %rd4;
+; SM90-NEXT: selp.f64 %rd10, %rd7, %rd9, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd19, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd11, %rd19, %rd5, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd19, %rd5;
+; SM90-NEXT: min.f64 %rd12, %rd19, %rd5;
; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd21, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
-; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd19, %rd5;
+; SM90-NEXT: selp.f64 %rd14, %rd11, %rd13, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
-; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd19};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd14};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
-; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
-; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
-; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
-; SM90-NEXT: mov.b64 %rd20, %rd1;
-; SM90-NEXT: mov.b64 %rd21, %rd2;
-; SM90-NEXT: @%p9 bra $L__BB281_1;
+; SM90-NEXT: xor.b64 %rd15, %rd2, %rd19;
+; SM90-NEXT: xor.b64 %rd16, %rd1, %rd18;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p7, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd18, %rd1;
+; SM90-NEXT: mov.b64 %rd19, %rd2;
+; SM90-NEXT: @%p7 bra $L__BB281_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -12610,8 +12604,8 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM90-LABEL: fminimum_acq_rel_v4f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<19>;
-; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-NEXT: .reg .pred %p<15>;
+; SM90-NEXT: .reg .b64 %rd<38>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
@@ -12624,85 +12618,77 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: mov.b128 {%rd34, %rd35}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd1, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p7, %rd2, -9223372036854775808;
; SM90-NEXT: $L__BB282_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
-; SM90-NEXT: min.f64 %rd11, %rd38, %rd1;
-; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd38, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
-; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
-; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
-; SM90-NEXT: min.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: setp.eq.b64 %p1, %rd34, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd11, %rd34, %rd1, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd34, %rd1;
+; SM90-NEXT: min.f64 %rd12, %rd34, %rd1;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd34, %rd1;
+; SM90-NEXT: selp.f64 %rd14, %rd11, %rd13, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd35, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd15, %rd35, %rd2, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd35, %rd2;
+; SM90-NEXT: min.f64 %rd16, %rd35, %rd2;
; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd39, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
-; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd35, %rd2;
+; SM90-NEXT: selp.f64 %rd18, %rd15, %rd17, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
-; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd35};
+; SM90-NEXT: mov.b128 swap, {%rd14, %rd18};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
-; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
-; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
-; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
-; SM90-NEXT: mov.b64 %rd38, %rd5;
-; SM90-NEXT: mov.b64 %rd39, %rd6;
-; SM90-NEXT: @%p9 bra $L__BB282_1;
+; SM90-NEXT: xor.b64 %rd19, %rd6, %rd35;
+; SM90-NEXT: xor.b64 %rd20, %rd5, %rd34;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p7, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd34, %rd5;
+; SM90-NEXT: mov.b64 %rd35, %rd6;
+; SM90-NEXT: @%p7 bra $L__BB282_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: mov.b64 %rd22, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
-; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd22};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd22};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: mov.b128 {%rd36, %rd37}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p12, %rd3, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p16, %rd4, -9223372036854775808;
; SM90-NEXT: $L__BB282_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
-; SM90-NEXT: min.f64 %rd25, %rd40, %rd3;
-; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
-; SM90-NEXT: setp.eq.b64 %p11, %rd40, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
-; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
-; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
-; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
-; SM90-NEXT: min.f64 %rd30, %rd41, %rd4;
-; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
-; SM90-NEXT: setp.eq.b64 %p15, %rd41, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
-; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
-; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
-; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: setp.eq.b64 %p8, %rd36, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd23, %rd36, %rd3, %p8;
+; SM90-NEXT: setp.nan.f64 %p9, %rd36, %rd3;
+; SM90-NEXT: min.f64 %rd24, %rd36, %rd3;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p9;
+; SM90-NEXT: setp.eq.f64 %p10, %rd36, %rd3;
+; SM90-NEXT: selp.f64 %rd26, %rd23, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd37, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd27, %rd37, %rd4, %p11;
+; SM90-NEXT: setp.nan.f64 %p12, %rd37, %rd4;
+; SM90-NEXT: min.f64 %rd28, %rd37, %rd4;
+; SM90-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd37, %rd4;
+; SM90-NEXT: selp.f64 %rd30, %rd27, %rd29, %p13;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd37};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd30};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
-; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
-; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
-; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
-; SM90-NEXT: mov.b64 %rd40, %rd7;
-; SM90-NEXT: mov.b64 %rd41, %rd8;
-; SM90-NEXT: @%p18 bra $L__BB282_3;
+; SM90-NEXT: xor.b64 %rd31, %rd8, %rd37;
+; SM90-NEXT: xor.b64 %rd32, %rd7, %rd36;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p14, %rd33, 0;
+; SM90-NEXT: mov.b64 %rd36, %rd7;
+; SM90-NEXT: mov.b64 %rd37, %rd8;
+; SM90-NEXT: @%p14 bra $L__BB282_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -12715,8 +12701,8 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM90-LABEL: fminimum_acq_rel_v8f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<37>;
-; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-NEXT: .reg .pred %p<29>;
+; SM90-NEXT: .reg .b64 %rd<74>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
@@ -12731,177 +12717,161 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: mov.b128 {%rd66, %rd67}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd5, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p7, %rd6, -9223372036854775808;
; SM90-NEXT: $L__BB283_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: mov.b64 %rd10, %rd75;
-; SM90-NEXT: mov.b64 %rd9, %rd74;
-; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
-; SM90-NEXT: min.f64 %rd19, %rd9, %rd5;
-; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
-; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: mov.b64 %rd10, %rd67;
+; SM90-NEXT: mov.b64 %rd9, %rd66;
+; SM90-NEXT: setp.eq.b64 %p1, %rd9, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd19, %rd9, %rd5, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd9, %rd5;
+; SM90-NEXT: min.f64 %rd20, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd22, %rd19, %rd21, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd10, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd23, %rd10, %rd6, %p4;
; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
; SM90-NEXT: min.f64 %rd24, %rd10, %rd6;
; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd10, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
-; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd26, %rd23, %rd25, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
-; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd26};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: mov.b128 {%rd66, %rd67}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
-; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
-; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
-; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
-; SM90-NEXT: @%p9 bra $L__BB283_1;
+; SM90-NEXT: xor.b64 %rd27, %rd67, %rd10;
+; SM90-NEXT: xor.b64 %rd28, %rd66, %rd9;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p7, %rd29, 0;
+; SM90-NEXT: @%p7 bra $L__BB283_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: mov.b64 %rd30, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
-; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: mov.b128 {%rd68, %rd69}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p12, %rd7, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p16, %rd8, -9223372036854775808;
; SM90-NEXT: $L__BB283_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: mov.b64 %rd12, %rd77;
-; SM90-NEXT: mov.b64 %rd11, %rd76;
-; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
-; SM90-NEXT: min.f64 %rd33, %rd11, %rd7;
-; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
-; SM90-NEXT: setp.eq.b64 %p11, %rd11, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
-; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
-; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
-; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
-; SM90-NEXT: min.f64 %rd38, %rd12, %rd8;
-; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
-; SM90-NEXT: setp.eq.b64 %p15, %rd12, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
-; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
-; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: mov.b64 %rd12, %rd69;
+; SM90-NEXT: mov.b64 %rd11, %rd68;
+; SM90-NEXT: setp.eq.b64 %p8, %rd11, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd31, %rd11, %rd7, %p8;
+; SM90-NEXT: setp.nan.f64 %p9, %rd11, %rd7;
+; SM90-NEXT: min.f64 %rd32, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd33, 0d7FF8000000000000, %rd32, %p9;
+; SM90-NEXT: setp.eq.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, %rd31, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd12, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd35, %rd12, %rd8, %p11;
+; SM90-NEXT: setp.nan.f64 %p12, %rd12, %rd8;
+; SM90-NEXT: min.f64 %rd36, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd37, 0d7FF8000000000000, %rd36, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd38, %rd35, %rd37, %p13;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd38};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: mov.b128 {%rd68, %rd69}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
-; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
-; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
-; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
-; SM90-NEXT: @%p18 bra $L__BB283_3;
+; SM90-NEXT: xor.b64 %rd39, %rd69, %rd12;
+; SM90-NEXT: xor.b64 %rd40, %rd68, %rd11;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p14, %rd41, 0;
+; SM90-NEXT: @%p14 bra $L__BB283_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
-; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
-; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: mov.b128 {%rd70, %rd71}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p21, %rd1, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p25, %rd2, -9223372036854775808;
; SM90-NEXT: $L__BB283_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
-; SM90-NEXT: min.f64 %rd47, %rd78, %rd1;
-; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
-; SM90-NEXT: setp.eq.b64 %p20, %rd78, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
-; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
-; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
-; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
-; SM90-NEXT: min.f64 %rd52, %rd79, %rd2;
-; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
-; SM90-NEXT: setp.eq.b64 %p24, %rd79, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
-; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
-; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
-; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: setp.eq.b64 %p15, %rd70, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd43, %rd70, %rd1, %p15;
+; SM90-NEXT: setp.nan.f64 %p16, %rd70, %rd1;
+; SM90-NEXT: min.f64 %rd44, %rd70, %rd1;
+; SM90-NEXT: selp.f64 %rd45, 0d7FF8000000000000, %rd44, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd70, %rd1;
+; SM90-NEXT: selp.f64 %rd46, %rd43, %rd45, %p17;
+; SM90-NEXT: setp.eq.b64 %p18, %rd71, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd47, %rd71, %rd2, %p18;
+; SM90-NEXT: setp.nan.f64 %p19, %rd71, %rd2;
+; SM90-NEXT: min.f64 %rd48, %rd71, %rd2;
+; SM90-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p19;
+; SM90-NEXT: setp.eq.f64 %p20, %rd71, %rd2;
+; SM90-NEXT: selp.f64 %rd50, %rd47, %rd49, %p20;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd70, %rd71};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd50};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
-; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
-; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
-; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
-; SM90-NEXT: mov.b64 %rd78, %rd13;
-; SM90-NEXT: mov.b64 %rd79, %rd14;
-; SM90-NEXT: @%p27 bra $L__BB283_5;
+; SM90-NEXT: xor.b64 %rd51, %rd14, %rd71;
+; SM90-NEXT: xor.b64 %rd52, %rd13, %rd70;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: setp.ne.b64 %p21, %rd53, 0;
+; SM90-NEXT: mov.b64 %rd70, %rd13;
+; SM90-NEXT: mov.b64 %rd71, %rd14;
+; SM90-NEXT: @%p21 bra $L__BB283_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
-; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: mov.b64 %rd54, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
-; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd54};
+; SM90-NEXT: mov.b128 swap, {%rd54, %rd54};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: mov.b128 {%rd72, %rd73}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p30, %rd3, -9223372036854775808;
-; SM90-NEXT: setp.eq.b64 %p34, %rd4, -9223372036854775808;
; SM90-NEXT: $L__BB283_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
-; SM90-NEXT: min.f64 %rd61, %rd80, %rd3;
-; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
-; SM90-NEXT: setp.eq.b64 %p29, %rd80, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
-; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
-; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
-; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
-; SM90-NEXT: min.f64 %rd66, %rd81, %rd4;
-; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
-; SM90-NEXT: setp.eq.b64 %p33, %rd81, -9223372036854775808;
-; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
-; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
-; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
-; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: setp.eq.b64 %p22, %rd72, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd55, %rd72, %rd3, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd72, %rd3;
+; SM90-NEXT: min.f64 %rd56, %rd72, %rd3;
+; SM90-NEXT: selp.f64 %rd57, 0d7FF8000000000000, %rd56, %p23;
+; SM90-NEXT: setp.eq.f64 %p24, %rd72, %rd3;
+; SM90-NEXT: selp.f64 %rd58, %rd55, %rd57, %p24;
+; SM90-NEXT: setp.eq.b64 %p25, %rd73, -9223372036854775808;
+; SM90-NEXT: selp.f64 %rd59, %rd73, %rd4, %p25;
+; SM90-NEXT: setp.nan.f64 %p26, %rd73, %rd4;
+; SM90-NEXT: min.f64 %rd60, %rd73, %rd4;
+; SM90-NEXT: selp.f64 %rd61, 0d7FF8000000000000, %rd60, %p26;
+; SM90-NEXT: setp.eq.f64 %p27, %rd73, %rd4;
+; SM90-NEXT: selp.f64 %rd62, %rd59, %rd61, %p27;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd72, %rd73};
+; SM90-NEXT: mov.b128 swap, {%rd58, %rd62};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
-; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
-; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
-; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
-; SM90-NEXT: mov.b64 %rd80, %rd15;
-; SM90-NEXT: mov.b64 %rd81, %rd16;
-; SM90-NEXT: @%p36 bra $L__BB283_7;
+; SM90-NEXT: xor.b64 %rd63, %rd16, %rd73;
+; SM90-NEXT: xor.b64 %rd64, %rd15, %rd72;
+; SM90-NEXT: or.b64 %rd65, %rd64, %rd63;
+; SM90-NEXT: setp.ne.b64 %p28, %rd65, 0;
+; SM90-NEXT: mov.b64 %rd72, %rd15;
+; SM90-NEXT: mov.b64 %rd73, %rd16;
+; SM90-NEXT: @%p28 bra $L__BB283_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd66, %rd67};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd68, %rd69};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12910,29 +12880,27 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM90-LABEL: fmaximum_acq_rel_v1f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<6>;
-; SM90-NEXT: .reg .b64 %rd<10>;
+; SM90-NEXT: .reg .pred %p<5>;
+; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd3];
-; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
+; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd1;
-; SM90-NEXT: max.f64 %rd4, %rd9, %rd1;
-; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
-; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2;
-; SM90-NEXT: selp.f64 %rd7, %rd1, %rd6, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd9, %rd8;
-; SM90-NEXT: setp.ne.b64 %p5, %rd2, %rd9;
-; SM90-NEXT: mov.b64 %rd9, %rd2;
-; SM90-NEXT: @%p5 bra $L__BB284_1;
+; SM90-NEXT: setp.eq.b64 %p1, %rd8, 0;
+; SM90-NEXT: selp.f64 %rd4, %rd8, %rd1, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd8, %rd1;
+; SM90-NEXT: max.f64 %rd5, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd8, %rd1;
+; SM90-NEXT: selp.f64 %rd7, %rd4, %rd6, %p3;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
+; SM90-NEXT: setp.ne.b64 %p4, %rd2, %rd8;
+; SM90-NEXT: mov.b64 %rd8, %rd2;
+; SM90-NEXT: @%p4 bra $L__BB284_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
@@ -12944,8 +12912,8 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM90-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<10>;
-; SM90-NEXT: .reg .b64 %rd<22>;
+; SM90-NEXT: .reg .pred %p<8>;
+; SM90-NEXT: .reg .b64 %rd<20>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
@@ -12957,42 +12925,38 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd6, %rd6};
; SM90-NEXT: mov.b128 swap, {%rd6, %rd6};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd20, %rd21}, dst;
+; SM90-NEXT: mov.b128 {%rd18, %rd19}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd4, 0;
-; SM90-NEXT: setp.eq.b64 %p7, %rd5, 0;
; SM90-NEXT: $L__BB285_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd20, %rd4;
-; SM90-NEXT: max.f64 %rd7, %rd20, %rd4;
-; SM90-NEXT: selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd20, 0;
-; SM90-NEXT: selp.f64 %rd9, %rd20, %rd8, %p2;
-; SM90-NEXT: selp.f64 %rd10, %rd4, %rd9, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd8, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd11, %rd10, %rd8, %p4;
-; SM90-NEXT: setp.nan.f64 %p5, %rd21, %rd5;
-; SM90-NEXT: max.f64 %rd12, %rd21, %rd5;
+; SM90-NEXT: setp.eq.b64 %p1, %rd18, 0;
+; SM90-NEXT: selp.f64 %rd7, %rd18, %rd4, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd18, %rd4;
+; SM90-NEXT: max.f64 %rd8, %rd18, %rd4;
+; SM90-NEXT: selp.f64 %rd9, 0d7FF8000000000000, %rd8, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd18, %rd4;
+; SM90-NEXT: selp.f64 %rd10, %rd7, %rd9, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd19, 0;
+; SM90-NEXT: selp.f64 %rd11, %rd19, %rd5, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd19, %rd5;
+; SM90-NEXT: max.f64 %rd12, %rd19, %rd5;
; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd21, 0;
-; SM90-NEXT: selp.f64 %rd14, %rd21, %rd13, %p6;
-; SM90-NEXT: selp.f64 %rd15, %rd5, %rd14, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd13, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd16, %rd15, %rd13, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd19, %rd5;
+; SM90-NEXT: selp.f64 %rd14, %rd11, %rd13, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd20, %rd21};
-; SM90-NEXT: mov.b128 swap, {%rd11, %rd16};
+; SM90-NEXT: mov.b128 cmp, {%rd18, %rd19};
+; SM90-NEXT: mov.b128 swap, {%rd10, %rd14};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd3], cmp, swap;
; SM90-NEXT: mov.b128 {%rd1, %rd2}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd17, %rd2, %rd21;
-; SM90-NEXT: xor.b64 %rd18, %rd1, %rd20;
-; SM90-NEXT: or.b64 %rd19, %rd18, %rd17;
-; SM90-NEXT: setp.ne.b64 %p9, %rd19, 0;
-; SM90-NEXT: mov.b64 %rd20, %rd1;
-; SM90-NEXT: mov.b64 %rd21, %rd2;
-; SM90-NEXT: @%p9 bra $L__BB285_1;
+; SM90-NEXT: xor.b64 %rd15, %rd2, %rd19;
+; SM90-NEXT: xor.b64 %rd16, %rd1, %rd18;
+; SM90-NEXT: or.b64 %rd17, %rd16, %rd15;
+; SM90-NEXT: setp.ne.b64 %p7, %rd17, 0;
+; SM90-NEXT: mov.b64 %rd18, %rd1;
+; SM90-NEXT: mov.b64 %rd19, %rd2;
+; SM90-NEXT: @%p7 bra $L__BB285_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
@@ -13004,8 +12968,8 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM90-LABEL: fmaximum_acq_rel_v4f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<19>;
-; SM90-NEXT: .reg .b64 %rd<42>;
+; SM90-NEXT: .reg .pred %p<15>;
+; SM90-NEXT: .reg .b64 %rd<38>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
@@ -13018,85 +12982,77 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd10, %rd10};
; SM90-NEXT: mov.b128 swap, {%rd10, %rd10};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd38, %rd39}, dst;
+; SM90-NEXT: mov.b128 {%rd34, %rd35}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd1, 0;
-; SM90-NEXT: setp.eq.b64 %p7, %rd2, 0;
; SM90-NEXT: $L__BB286_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p1, %rd38, %rd1;
-; SM90-NEXT: max.f64 %rd11, %rd38, %rd1;
-; SM90-NEXT: selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd38, 0;
-; SM90-NEXT: selp.f64 %rd13, %rd38, %rd12, %p2;
-; SM90-NEXT: selp.f64 %rd14, %rd1, %rd13, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd12, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd15, %rd14, %rd12, %p4;
-; SM90-NEXT: setp.nan.f64 %p5, %rd39, %rd2;
-; SM90-NEXT: max.f64 %rd16, %rd39, %rd2;
+; SM90-NEXT: setp.eq.b64 %p1, %rd34, 0;
+; SM90-NEXT: selp.f64 %rd11, %rd34, %rd1, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd34, %rd1;
+; SM90-NEXT: max.f64 %rd12, %rd34, %rd1;
+; SM90-NEXT: selp.f64 %rd13, 0d7FF8000000000000, %rd12, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd34, %rd1;
+; SM90-NEXT: selp.f64 %rd14, %rd11, %rd13, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd35, 0;
+; SM90-NEXT: selp.f64 %rd15, %rd35, %rd2, %p4;
+; SM90-NEXT: setp.nan.f64 %p5, %rd35, %rd2;
+; SM90-NEXT: max.f64 %rd16, %rd35, %rd2;
; SM90-NEXT: selp.f64 %rd17, 0d7FF8000000000000, %rd16, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd39, 0;
-; SM90-NEXT: selp.f64 %rd18, %rd39, %rd17, %p6;
-; SM90-NEXT: selp.f64 %rd19, %rd2, %rd18, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd17, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd20, %rd19, %rd17, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd35, %rd2;
+; SM90-NEXT: selp.f64 %rd18, %rd15, %rd17, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd38, %rd39};
-; SM90-NEXT: mov.b128 swap, {%rd15, %rd20};
+; SM90-NEXT: mov.b128 cmp, {%rd34, %rd35};
+; SM90-NEXT: mov.b128 swap, {%rd14, %rd18};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9], cmp, swap;
; SM90-NEXT: mov.b128 {%rd5, %rd6}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd21, %rd6, %rd39;
-; SM90-NEXT: xor.b64 %rd22, %rd5, %rd38;
-; SM90-NEXT: or.b64 %rd23, %rd22, %rd21;
-; SM90-NEXT: setp.ne.b64 %p9, %rd23, 0;
-; SM90-NEXT: mov.b64 %rd38, %rd5;
-; SM90-NEXT: mov.b64 %rd39, %rd6;
-; SM90-NEXT: @%p9 bra $L__BB286_1;
+; SM90-NEXT: xor.b64 %rd19, %rd6, %rd35;
+; SM90-NEXT: xor.b64 %rd20, %rd5, %rd34;
+; SM90-NEXT: or.b64 %rd21, %rd20, %rd19;
+; SM90-NEXT: setp.ne.b64 %p7, %rd21, 0;
+; SM90-NEXT: mov.b64 %rd34, %rd5;
+; SM90-NEXT: mov.b64 %rd35, %rd6;
+; SM90-NEXT: @%p7 bra $L__BB286_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: mov.b64 %rd24, 0;
+; SM90-NEXT: mov.b64 %rd22, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd24, %rd24};
-; SM90-NEXT: mov.b128 swap, {%rd24, %rd24};
+; SM90-NEXT: mov.b128 cmp, {%rd22, %rd22};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd22};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd40, %rd41}, dst;
+; SM90-NEXT: mov.b128 {%rd36, %rd37}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p12, %rd3, 0;
-; SM90-NEXT: setp.eq.b64 %p16, %rd4, 0;
; SM90-NEXT: $L__BB286_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p10, %rd40, %rd3;
-; SM90-NEXT: max.f64 %rd25, %rd40, %rd3;
-; SM90-NEXT: selp.f64 %rd26, 0d7FF8000000000000, %rd25, %p10;
-; SM90-NEXT: setp.eq.b64 %p11, %rd40, 0;
-; SM90-NEXT: selp.f64 %rd27, %rd40, %rd26, %p11;
-; SM90-NEXT: selp.f64 %rd28, %rd3, %rd27, %p12;
-; SM90-NEXT: setp.eq.f64 %p13, %rd26, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd29, %rd28, %rd26, %p13;
-; SM90-NEXT: setp.nan.f64 %p14, %rd41, %rd4;
-; SM90-NEXT: max.f64 %rd30, %rd41, %rd4;
-; SM90-NEXT: selp.f64 %rd31, 0d7FF8000000000000, %rd30, %p14;
-; SM90-NEXT: setp.eq.b64 %p15, %rd41, 0;
-; SM90-NEXT: selp.f64 %rd32, %rd41, %rd31, %p15;
-; SM90-NEXT: selp.f64 %rd33, %rd4, %rd32, %p16;
-; SM90-NEXT: setp.eq.f64 %p17, %rd31, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd34, %rd33, %rd31, %p17;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd40, %rd41};
-; SM90-NEXT: mov.b128 swap, {%rd29, %rd34};
+; SM90-NEXT: setp.eq.b64 %p8, %rd36, 0;
+; SM90-NEXT: selp.f64 %rd23, %rd36, %rd3, %p8;
+; SM90-NEXT: setp.nan.f64 %p9, %rd36, %rd3;
+; SM90-NEXT: max.f64 %rd24, %rd36, %rd3;
+; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p9;
+; SM90-NEXT: setp.eq.f64 %p10, %rd36, %rd3;
+; SM90-NEXT: selp.f64 %rd26, %rd23, %rd25, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd37, 0;
+; SM90-NEXT: selp.f64 %rd27, %rd37, %rd4, %p11;
+; SM90-NEXT: setp.nan.f64 %p12, %rd37, %rd4;
+; SM90-NEXT: max.f64 %rd28, %rd37, %rd4;
+; SM90-NEXT: selp.f64 %rd29, 0d7FF8000000000000, %rd28, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd37, %rd4;
+; SM90-NEXT: selp.f64 %rd30, %rd27, %rd29, %p13;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd36, %rd37};
+; SM90-NEXT: mov.b128 swap, {%rd26, %rd30};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd9+16], cmp, swap;
; SM90-NEXT: mov.b128 {%rd7, %rd8}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd35, %rd8, %rd41;
-; SM90-NEXT: xor.b64 %rd36, %rd7, %rd40;
-; SM90-NEXT: or.b64 %rd37, %rd36, %rd35;
-; SM90-NEXT: setp.ne.b64 %p18, %rd37, 0;
-; SM90-NEXT: mov.b64 %rd40, %rd7;
-; SM90-NEXT: mov.b64 %rd41, %rd8;
-; SM90-NEXT: @%p18 bra $L__BB286_3;
+; SM90-NEXT: xor.b64 %rd31, %rd8, %rd37;
+; SM90-NEXT: xor.b64 %rd32, %rd7, %rd36;
+; SM90-NEXT: or.b64 %rd33, %rd32, %rd31;
+; SM90-NEXT: setp.ne.b64 %p14, %rd33, 0;
+; SM90-NEXT: mov.b64 %rd36, %rd7;
+; SM90-NEXT: mov.b64 %rd37, %rd8;
+; SM90-NEXT: @%p14 bra $L__BB286_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
@@ -13109,8 +13065,8 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM90-LABEL: fmaximum_acq_rel_v8f64_global_cta(
; SM90: {
-; SM90-NEXT: .reg .pred %p<37>;
-; SM90-NEXT: .reg .b64 %rd<82>;
+; SM90-NEXT: .reg .pred %p<29>;
+; SM90-NEXT: .reg .b64 %rd<74>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
; SM90-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
@@ -13125,177 +13081,161 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: mov.b128 cmp, {%rd18, %rd18};
; SM90-NEXT: mov.b128 swap, {%rd18, %rd18};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: mov.b128 {%rd66, %rd67}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p3, %rd5, 0;
-; SM90-NEXT: setp.eq.b64 %p7, %rd6, 0;
; SM90-NEXT: $L__BB287_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: mov.b64 %rd10, %rd75;
-; SM90-NEXT: mov.b64 %rd9, %rd74;
-; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd5;
-; SM90-NEXT: max.f64 %rd19, %rd9, %rd5;
-; SM90-NEXT: selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p1;
-; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0;
-; SM90-NEXT: selp.f64 %rd21, %rd9, %rd20, %p2;
-; SM90-NEXT: selp.f64 %rd22, %rd5, %rd21, %p3;
-; SM90-NEXT: setp.eq.f64 %p4, %rd20, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd23, %rd22, %rd20, %p4;
+; SM90-NEXT: mov.b64 %rd10, %rd67;
+; SM90-NEXT: mov.b64 %rd9, %rd66;
+; SM90-NEXT: setp.eq.b64 %p1, %rd9, 0;
+; SM90-NEXT: selp.f64 %rd19, %rd9, %rd5, %p1;
+; SM90-NEXT: setp.nan.f64 %p2, %rd9, %rd5;
+; SM90-NEXT: max.f64 %rd20, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd21, 0d7FF8000000000000, %rd20, %p2;
+; SM90-NEXT: setp.eq.f64 %p3, %rd9, %rd5;
+; SM90-NEXT: selp.f64 %rd22, %rd19, %rd21, %p3;
+; SM90-NEXT: setp.eq.b64 %p4, %rd10, 0;
+; SM90-NEXT: selp.f64 %rd23, %rd10, %rd6, %p4;
; SM90-NEXT: setp.nan.f64 %p5, %rd10, %rd6;
; SM90-NEXT: max.f64 %rd24, %rd10, %rd6;
; SM90-NEXT: selp.f64 %rd25, 0d7FF8000000000000, %rd24, %p5;
-; SM90-NEXT: setp.eq.b64 %p6, %rd10, 0;
-; SM90-NEXT: selp.f64 %rd26, %rd10, %rd25, %p6;
-; SM90-NEXT: selp.f64 %rd27, %rd6, %rd26, %p7;
-; SM90-NEXT: setp.eq.f64 %p8, %rd25, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd28, %rd27, %rd25, %p8;
+; SM90-NEXT: setp.eq.f64 %p6, %rd10, %rd6;
+; SM90-NEXT: selp.f64 %rd26, %rd23, %rd25, %p6;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
; SM90-NEXT: mov.b128 cmp, {%rd9, %rd10};
-; SM90-NEXT: mov.b128 swap, {%rd23, %rd28};
+; SM90-NEXT: mov.b128 swap, {%rd22, %rd26};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd74, %rd75}, dst;
+; SM90-NEXT: mov.b128 {%rd66, %rd67}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd29, %rd75, %rd10;
-; SM90-NEXT: xor.b64 %rd30, %rd74, %rd9;
-; SM90-NEXT: or.b64 %rd31, %rd30, %rd29;
-; SM90-NEXT: setp.ne.b64 %p9, %rd31, 0;
-; SM90-NEXT: @%p9 bra $L__BB287_1;
+; SM90-NEXT: xor.b64 %rd27, %rd67, %rd10;
+; SM90-NEXT: xor.b64 %rd28, %rd66, %rd9;
+; SM90-NEXT: or.b64 %rd29, %rd28, %rd27;
+; SM90-NEXT: setp.ne.b64 %p7, %rd29, 0;
+; SM90-NEXT: @%p7 bra $L__BB287_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: mov.b64 %rd32, 0;
+; SM90-NEXT: mov.b64 %rd30, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd32, %rd32};
-; SM90-NEXT: mov.b128 swap, {%rd32, %rd32};
+; SM90-NEXT: mov.b128 cmp, {%rd30, %rd30};
+; SM90-NEXT: mov.b128 swap, {%rd30, %rd30};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: mov.b128 {%rd68, %rd69}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p12, %rd7, 0;
-; SM90-NEXT: setp.eq.b64 %p16, %rd8, 0;
; SM90-NEXT: $L__BB287_3: // %atomicrmw.start6
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: mov.b64 %rd12, %rd77;
-; SM90-NEXT: mov.b64 %rd11, %rd76;
-; SM90-NEXT: setp.nan.f64 %p10, %rd11, %rd7;
-; SM90-NEXT: max.f64 %rd33, %rd11, %rd7;
-; SM90-NEXT: selp.f64 %rd34, 0d7FF8000000000000, %rd33, %p10;
-; SM90-NEXT: setp.eq.b64 %p11, %rd11, 0;
-; SM90-NEXT: selp.f64 %rd35, %rd11, %rd34, %p11;
-; SM90-NEXT: selp.f64 %rd36, %rd7, %rd35, %p12;
-; SM90-NEXT: setp.eq.f64 %p13, %rd34, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd37, %rd36, %rd34, %p13;
-; SM90-NEXT: setp.nan.f64 %p14, %rd12, %rd8;
-; SM90-NEXT: max.f64 %rd38, %rd12, %rd8;
-; SM90-NEXT: selp.f64 %rd39, 0d7FF8000000000000, %rd38, %p14;
-; SM90-NEXT: setp.eq.b64 %p15, %rd12, 0;
-; SM90-NEXT: selp.f64 %rd40, %rd12, %rd39, %p15;
-; SM90-NEXT: selp.f64 %rd41, %rd8, %rd40, %p16;
-; SM90-NEXT: setp.eq.f64 %p17, %rd39, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd42, %rd41, %rd39, %p17;
+; SM90-NEXT: mov.b64 %rd12, %rd69;
+; SM90-NEXT: mov.b64 %rd11, %rd68;
+; SM90-NEXT: setp.eq.b64 %p8, %rd11, 0;
+; SM90-NEXT: selp.f64 %rd31, %rd11, %rd7, %p8;
+; SM90-NEXT: setp.nan.f64 %p9, %rd11, %rd7;
+; SM90-NEXT: max.f64 %rd32, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd33, 0d7FF8000000000000, %rd32, %p9;
+; SM90-NEXT: setp.eq.f64 %p10, %rd11, %rd7;
+; SM90-NEXT: selp.f64 %rd34, %rd31, %rd33, %p10;
+; SM90-NEXT: setp.eq.b64 %p11, %rd12, 0;
+; SM90-NEXT: selp.f64 %rd35, %rd12, %rd8, %p11;
+; SM90-NEXT: setp.nan.f64 %p12, %rd12, %rd8;
+; SM90-NEXT: max.f64 %rd36, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd37, 0d7FF8000000000000, %rd36, %p12;
+; SM90-NEXT: setp.eq.f64 %p13, %rd12, %rd8;
+; SM90-NEXT: selp.f64 %rd38, %rd35, %rd37, %p13;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
; SM90-NEXT: mov.b128 cmp, {%rd11, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd37, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd34, %rd38};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd76, %rd77}, dst;
+; SM90-NEXT: mov.b128 {%rd68, %rd69}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd43, %rd77, %rd12;
-; SM90-NEXT: xor.b64 %rd44, %rd76, %rd11;
-; SM90-NEXT: or.b64 %rd45, %rd44, %rd43;
-; SM90-NEXT: setp.ne.b64 %p18, %rd45, 0;
-; SM90-NEXT: @%p18 bra $L__BB287_3;
+; SM90-NEXT: xor.b64 %rd39, %rd69, %rd12;
+; SM90-NEXT: xor.b64 %rd40, %rd68, %rd11;
+; SM90-NEXT: or.b64 %rd41, %rd40, %rd39;
+; SM90-NEXT: setp.ne.b64 %p14, %rd41, 0;
+; SM90-NEXT: @%p14 bra $L__BB287_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end5
-; SM90-NEXT: mov.b64 %rd46, 0;
+; SM90-NEXT: mov.b64 %rd42, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd46, %rd46};
-; SM90-NEXT: mov.b128 swap, {%rd46, %rd46};
+; SM90-NEXT: mov.b128 cmp, {%rd42, %rd42};
+; SM90-NEXT: mov.b128 swap, {%rd42, %rd42};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd78, %rd79}, dst;
+; SM90-NEXT: mov.b128 {%rd70, %rd71}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p21, %rd1, 0;
-; SM90-NEXT: setp.eq.b64 %p25, %rd2, 0;
; SM90-NEXT: $L__BB287_5: // %atomicrmw.start15
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p19, %rd78, %rd1;
-; SM90-NEXT: max.f64 %rd47, %rd78, %rd1;
-; SM90-NEXT: selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p19;
-; SM90-NEXT: setp.eq.b64 %p20, %rd78, 0;
-; SM90-NEXT: selp.f64 %rd49, %rd78, %rd48, %p20;
-; SM90-NEXT: selp.f64 %rd50, %rd1, %rd49, %p21;
-; SM90-NEXT: setp.eq.f64 %p22, %rd48, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd51, %rd50, %rd48, %p22;
-; SM90-NEXT: setp.nan.f64 %p23, %rd79, %rd2;
-; SM90-NEXT: max.f64 %rd52, %rd79, %rd2;
-; SM90-NEXT: selp.f64 %rd53, 0d7FF8000000000000, %rd52, %p23;
-; SM90-NEXT: setp.eq.b64 %p24, %rd79, 0;
-; SM90-NEXT: selp.f64 %rd54, %rd79, %rd53, %p24;
-; SM90-NEXT: selp.f64 %rd55, %rd2, %rd54, %p25;
-; SM90-NEXT: setp.eq.f64 %p26, %rd53, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd56, %rd55, %rd53, %p26;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd78, %rd79};
-; SM90-NEXT: mov.b128 swap, {%rd51, %rd56};
+; SM90-NEXT: setp.eq.b64 %p15, %rd70, 0;
+; SM90-NEXT: selp.f64 %rd43, %rd70, %rd1, %p15;
+; SM90-NEXT: setp.nan.f64 %p16, %rd70, %rd1;
+; SM90-NEXT: max.f64 %rd44, %rd70, %rd1;
+; SM90-NEXT: selp.f64 %rd45, 0d7FF8000000000000, %rd44, %p16;
+; SM90-NEXT: setp.eq.f64 %p17, %rd70, %rd1;
+; SM90-NEXT: selp.f64 %rd46, %rd43, %rd45, %p17;
+; SM90-NEXT: setp.eq.b64 %p18, %rd71, 0;
+; SM90-NEXT: selp.f64 %rd47, %rd71, %rd2, %p18;
+; SM90-NEXT: setp.nan.f64 %p19, %rd71, %rd2;
+; SM90-NEXT: max.f64 %rd48, %rd71, %rd2;
+; SM90-NEXT: selp.f64 %rd49, 0d7FF8000000000000, %rd48, %p19;
+; SM90-NEXT: setp.eq.f64 %p20, %rd71, %rd2;
+; SM90-NEXT: selp.f64 %rd50, %rd47, %rd49, %p20;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd70, %rd71};
+; SM90-NEXT: mov.b128 swap, {%rd46, %rd50};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+32], cmp, swap;
; SM90-NEXT: mov.b128 {%rd13, %rd14}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd57, %rd14, %rd79;
-; SM90-NEXT: xor.b64 %rd58, %rd13, %rd78;
-; SM90-NEXT: or.b64 %rd59, %rd58, %rd57;
-; SM90-NEXT: setp.ne.b64 %p27, %rd59, 0;
-; SM90-NEXT: mov.b64 %rd78, %rd13;
-; SM90-NEXT: mov.b64 %rd79, %rd14;
-; SM90-NEXT: @%p27 bra $L__BB287_5;
+; SM90-NEXT: xor.b64 %rd51, %rd14, %rd71;
+; SM90-NEXT: xor.b64 %rd52, %rd13, %rd70;
+; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
+; SM90-NEXT: setp.ne.b64 %p21, %rd53, 0;
+; SM90-NEXT: mov.b64 %rd70, %rd13;
+; SM90-NEXT: mov.b64 %rd71, %rd14;
+; SM90-NEXT: @%p21 bra $L__BB287_5;
; SM90-NEXT: // %bb.6: // %atomicrmw.end14
-; SM90-NEXT: mov.b64 %rd60, 0;
+; SM90-NEXT: mov.b64 %rd54, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd60, %rd60};
-; SM90-NEXT: mov.b128 swap, {%rd60, %rd60};
+; SM90-NEXT: mov.b128 cmp, {%rd54, %rd54};
+; SM90-NEXT: mov.b128 swap, {%rd54, %rd54};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd80, %rd81}, dst;
+; SM90-NEXT: mov.b128 {%rd72, %rd73}, dst;
; SM90-NEXT: }
-; SM90-NEXT: setp.eq.b64 %p30, %rd3, 0;
-; SM90-NEXT: setp.eq.b64 %p34, %rd4, 0;
; SM90-NEXT: $L__BB287_7: // %atomicrmw.start20
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: setp.nan.f64 %p28, %rd80, %rd3;
-; SM90-NEXT: max.f64 %rd61, %rd80, %rd3;
-; SM90-NEXT: selp.f64 %rd62, 0d7FF8000000000000, %rd61, %p28;
-; SM90-NEXT: setp.eq.b64 %p29, %rd80, 0;
-; SM90-NEXT: selp.f64 %rd63, %rd80, %rd62, %p29;
-; SM90-NEXT: selp.f64 %rd64, %rd3, %rd63, %p30;
-; SM90-NEXT: setp.eq.f64 %p31, %rd62, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd65, %rd64, %rd62, %p31;
-; SM90-NEXT: setp.nan.f64 %p32, %rd81, %rd4;
-; SM90-NEXT: max.f64 %rd66, %rd81, %rd4;
-; SM90-NEXT: selp.f64 %rd67, 0d7FF8000000000000, %rd66, %p32;
-; SM90-NEXT: setp.eq.b64 %p33, %rd81, 0;
-; SM90-NEXT: selp.f64 %rd68, %rd81, %rd67, %p33;
-; SM90-NEXT: selp.f64 %rd69, %rd4, %rd68, %p34;
-; SM90-NEXT: setp.eq.f64 %p35, %rd67, 0d0000000000000000;
-; SM90-NEXT: selp.f64 %rd70, %rd69, %rd67, %p35;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd80, %rd81};
-; SM90-NEXT: mov.b128 swap, {%rd65, %rd70};
+; SM90-NEXT: setp.eq.b64 %p22, %rd72, 0;
+; SM90-NEXT: selp.f64 %rd55, %rd72, %rd3, %p22;
+; SM90-NEXT: setp.nan.f64 %p23, %rd72, %rd3;
+; SM90-NEXT: max.f64 %rd56, %rd72, %rd3;
+; SM90-NEXT: selp.f64 %rd57, 0d7FF8000000000000, %rd56, %p23;
+; SM90-NEXT: setp.eq.f64 %p24, %rd72, %rd3;
+; SM90-NEXT: selp.f64 %rd58, %rd55, %rd57, %p24;
+; SM90-NEXT: setp.eq.b64 %p25, %rd73, 0;
+; SM90-NEXT: selp.f64 %rd59, %rd73, %rd4, %p25;
+; SM90-NEXT: setp.nan.f64 %p26, %rd73, %rd4;
+; SM90-NEXT: max.f64 %rd60, %rd73, %rd4;
+; SM90-NEXT: selp.f64 %rd61, 0d7FF8000000000000, %rd60, %p26;
+; SM90-NEXT: setp.eq.f64 %p27, %rd73, %rd4;
+; SM90-NEXT: selp.f64 %rd62, %rd59, %rd61, %p27;
+; SM90-NEXT: {
+; SM90-NEXT: .reg .b128 cmp, swap, dst;
+; SM90-NEXT: mov.b128 cmp, {%rd72, %rd73};
+; SM90-NEXT: mov.b128 swap, {%rd58, %rd62};
; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd17+48], cmp, swap;
; SM90-NEXT: mov.b128 {%rd15, %rd16}, dst;
; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd71, %rd16, %rd81;
-; SM90-NEXT: xor.b64 %rd72, %rd15, %rd80;
-; SM90-NEXT: or.b64 %rd73, %rd72, %rd71;
-; SM90-NEXT: setp.ne.b64 %p36, %rd73, 0;
-; SM90-NEXT: mov.b64 %rd80, %rd15;
-; SM90-NEXT: mov.b64 %rd81, %rd16;
-; SM90-NEXT: @%p36 bra $L__BB287_7;
+; SM90-NEXT: xor.b64 %rd63, %rd16, %rd73;
+; SM90-NEXT: xor.b64 %rd64, %rd15, %rd72;
+; SM90-NEXT: or.b64 %rd65, %rd64, %rd63;
+; SM90-NEXT: setp.ne.b64 %p28, %rd65, 0;
+; SM90-NEXT: mov.b64 %rd72, %rd15;
+; SM90-NEXT: mov.b64 %rd73, %rd16;
+; SM90-NEXT: @%p28 bra $L__BB287_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd74, %rd75};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd76, %rd77};
+; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd66, %rd67};
+; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd68, %rd69};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index b573a11f1e4e2..12dd0f4088f95 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -3,37 +3,36 @@
target triple = "nvptx64-nvidia-cuda"
-; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW and wraps
-; the old value back into a single-lane vector.
-define <1 x float> @fadd_v1f32_elementwise(ptr %addr, <1 x float> %val) {
+; A <1 x T> elementwise atomicrmw collapses directly to a scalar RMW.
+; Shared-memory f32 atom.add preserves denormals, matching the default mode.
+define <1 x float> @fadd_v1f32_elementwise(ptr addrspace(3) %addr, <1 x float> %val) {
; CHECK-LABEL: @fadd_v1f32_elementwise(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[LANE_VAL:%.*]] = extractelement <1 x float> [[VAL:%.*]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR:%.*]], float [[LANE_VAL]] monotonic, align 4
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr addrspace(3) [[ADDR:%.*]], float [[LANE_VAL]] monotonic, align 4
; CHECK-NEXT: [[LANE_OLD:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
; CHECK-NEXT: ret <1 x float> [[LANE_OLD]]
;
entry:
- %old = atomicrmw elementwise fadd ptr %addr, <1 x float> %val monotonic
+ %old = atomicrmw elementwise fadd ptr addrspace(3) %addr, <1 x float> %val monotonic
ret <1 x float> %old
}
-; Elementwise expansion: scalar f32 atomics are legal, so the vector is split
-; into scalar atomicrmw instructions.
-define <2 x float> @fadd_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; A <2 x float> elementwise fadd in shared memory splits into native scalar atomics.
+define <2 x float> @fadd_v2f32_elementwise(ptr addrspace(3) %addr, <2 x float> %val) {
; CHECK-LABEL: @fadd_v2f32_elementwise(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
-; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr [[ADDR]], float [[LO_VAL]] monotonic, align 8
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw fadd ptr addrspace(3) [[ADDR]], float [[LO_VAL]] monotonic, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw fadd ptr addrspace(3) [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP1]], i64 1
; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
;
entry:
- %old = atomicrmw elementwise fadd ptr %addr, <2 x float> %val monotonic
+ %old = atomicrmw elementwise fadd ptr addrspace(3) %addr, <2 x float> %val monotonic
ret <2 x float> %old
}
@@ -56,20 +55,20 @@ entry:
}
; Volatile is preserved while splitting elementwise operations.
-define <2 x float> @fadd_volatile_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+define <2 x float> @fadd_volatile_v2f32_elementwise(ptr addrspace(3) %addr, <2 x float> %val) {
; CHECK-LABEL: @fadd_volatile_v2f32_elementwise(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
-; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw volatile fadd ptr [[ADDR]], float [[LO_VAL]] monotonic, align 8
-; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile fadd ptr [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw volatile fadd ptr addrspace(3) [[ADDR]], float [[LO_VAL]] monotonic, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw volatile fadd ptr addrspace(3) [[HI_PTR]], float [[HI_VAL]] monotonic, align 4
; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP1]], i64 1
; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
;
entry:
- %old = atomicrmw volatile elementwise fadd ptr %addr, <2 x float> %val monotonic
+ %old = atomicrmw volatile elementwise fadd ptr addrspace(3) %addr, <2 x float> %val monotonic
ret <2 x float> %old
}
>From 6235b491f1a0bc8681e1010944e1b45fe98616c9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 16:51:13 +0000
Subject: [PATCH 10/15] expand by default
---
llvm/include/llvm/CodeGen/TargetLowering.h | 9 +++++---
.../Mips/expand-atomic-rmw-elementwise.ll | 21 +++++++++++++++++++
.../X86/expand-atomic-rmw-elementwise.ll | 8 ++++---
3 files changed, 32 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 6ceb3150e7610..720250bb6c249 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2485,11 +2485,14 @@ class LLVM_ABI TargetLoweringBase {
}
/// Returns how the IR-level AtomicExpand pass should expand the given
- /// AtomicRMW, if at all. Default is to never expand.
+ /// AtomicRMW, if at all. By default, elementwise operations are expanded and
+ /// floating-point operations use are expanded to a compare-exchange loop.
virtual AtomicExpansionKind
shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
- return RMW->isFloatingPointOperation() ?
- AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None;
+ if (RMW->isElementwise())
+ return AtomicExpansionKind::Expand;
+ return RMW->isFloatingPointOperation() ? AtomicExpansionKind::CmpXChg
+ : AtomicExpansionKind::None;
}
/// Returns how the given atomic atomicrmw should be cast by the IR-level
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
new file mode 100644
index 0000000000000..b2b30dfe24d06
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -mtriple=mips64-mti-linux-gnu -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s
+
+; MIPS uses TargetLowering's default AtomicRMW expansion, so an elementwise
+; atomicrmw is expanded into a compare-exchange loop.
+define <2 x i32> @atomicrmw_add_elementwise(ptr %ptr, <2 x i32> %value) {
+; CHECK-LABEL: define <2 x i32> @atomicrmw_add_elementwise(
+; CHECK-SAME: ptr [[PTR:%.*]], <2 x i32> [[VALUE:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VALUE:%.*]] = extractelement <2 x i32> [[VALUE]], i64 0
+; CHECK-NEXT: [[HI_VALUE:%.*]] = extractelement <2 x i32> [[VALUE]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[LO_OLD:%.*]] = atomicrmw add ptr [[PTR]], i32 [[LO_VALUE]] monotonic, align 8
+; CHECK-NEXT: [[HI_OLD:%.*]] = atomicrmw add ptr [[HI_PTR]], i32 [[HI_VALUE]] monotonic, align 4
+; CHECK-NEXT: [[RESULT:%.*]] = insertelement <2 x i32> poison, i32 [[LO_OLD]], i64 0
+; CHECK-NEXT: [[RESULT1:%.*]] = insertelement <2 x i32> [[RESULT]], i32 [[HI_OLD]], i64 1
+; CHECK-NEXT: ret <2 x i32> [[RESULT1]]
+;
+entry:
+ %old = atomicrmw elementwise add ptr %ptr, <2 x i32> %value monotonic
+ ret <2 x i32> %old
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
index 9c5b7857fe1b9..d0c6a98025caa 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-elementwise.ll
@@ -1,9 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s
-; By default, elementwise atomicrmws are treated the same as the non-elementwise versions and are not expanded.
-; Targets in the future can take advantage of elementwise expansion by returning AtomicExpansionKind::Expand from shouldExpandAtomicRMWInIR.
-; Elementwise atomics work on integer vectors, but normal atomics don't.
+; By default, elementwise atomicrmws are expanded because it is not generally
+; safe to treat an elementwise atomicrmw as a whole-vector atomicrmw.
+;
+; FIXME: X86ISelLowering and many other backends override
+; shouldExpandAtomicRMWInIR and currently do not expand elementwise atomicrmws.
define <4 x i32> @atomicrmw_add_elementwise(ptr %p, <4 x i32> %v) {
; CHECK-LABEL: define <4 x i32> @atomicrmw_add_elementwise(
>From 7ea2ca28338a414af3fc6d49745d0adda94a2dcf Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 18:38:23 +0000
Subject: [PATCH 11/15] nfc cleanup and get rid of shouldCastAtomicLoad
override
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 7 ++-----
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 11 -----------
.../NVPTX/expand-atomic-rmw-elementwise.ll | 9 +++------
3 files changed, 5 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index d6930235a387c..00b8ff5ceedb2 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -415,11 +415,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
// - into a load if it is idempotent
// - into a Cmpxchg/LL-SC loop otherwise
// we try them in that order.
- if (IsElementwiseExpand)
- MadeChange |= tryExpandAtomicRMW(RMWI);
- else
- MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
- tryExpandAtomicRMW(RMWI);
+ MadeChange |= (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) ||
+ tryExpandAtomicRMW(RMWI);
return MadeChange;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 2ec8994e0f1ed..a7f0abc6eda79 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -146,17 +146,6 @@ class NVPTXTargetLowering : public TargetLowering {
bool isCheapToSpeculateCtlz(Type *Ty) const override { return true; }
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override {
- Type *Ty = LI->getType();
- // SelectionDAG can split vector computations, but not vector atomic loads.
- // Splitting those loads into per-lane atomics would change the atomicity,
- // so cast them to a single same-width integer atomic load instead.
- if (Ty->isVectorTy()) {
- TypeSize SizeInBits = LI->getDataLayout().getTypeStoreSizeInBits(Ty);
- if (!SizeInBits.isScalable() &&
- SizeInBits.getFixedValue() <= getMaxAtomicSizeInBitsSupported())
- return AtomicExpansionKind::CastToInteger;
- }
-
return AtomicExpansionKind::None;
}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index 12dd0f4088f95..14825faa144c4 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -129,8 +129,7 @@ entry:
define <8 x half> @fmin_v8f16_elementwise(ptr %addr, <8 x half> %val) {
; CHECK-LABEL: @fmin_v8f16_elementwise(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr [[ADDR:%.*]] monotonic, align 16
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast i128 [[TMP0]] to <8 x half>
+; CHECK-NEXT: [[TMP1:%.*]] = load atomic <8 x half>, ptr [[ADDR:%.*]] monotonic, align 16
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi <8 x half> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
@@ -239,8 +238,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; CHECK-NEXT: [[LO_VAL:%.*]] = shufflevector <8 x i32> [[VAL:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[HI_VAL:%.*]] = shufflevector <8 x i32> [[VAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[TMP0:%.*]] = load atomic i128, ptr addrspace(1) [[ADDR]] syncscope("block") monotonic, align 32
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast i128 [[TMP0]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = load atomic <4 x i32>, ptr addrspace(1) [[ADDR]] syncscope("block") monotonic, align 32
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi <4 x i32> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
@@ -254,8 +252,7 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; CHECK-NEXT: [[TMP6]] = bitcast i128 [[NEWLOADED]] to <4 x i32>
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
-; CHECK-NEXT: [[TMP7:%.*]] = load atomic i128, ptr addrspace(1) [[HI_PTR]] syncscope("block") monotonic, align 16
-; CHECK-NEXT: [[TMP8:%.*]] = bitcast i128 [[TMP7]] to <4 x i32>
+; CHECK-NEXT: [[TMP8:%.*]] = load atomic <4 x i32>, ptr addrspace(1) [[HI_PTR]] syncscope("block") monotonic, align 16
; CHECK-NEXT: br label [[ATOMICRMW_START2:%.*]]
; CHECK: atomicrmw.start2:
; CHECK-NEXT: [[LOADED3:%.*]] = phi <4 x i32> [ [[TMP8]], [[ATOMICRMW_END]] ], [ [[TMP13:%.*]], [[ATOMICRMW_START2]] ]
>From b08a77635bd2ce5e70e34e6ba4f4e4192c7b5cfc Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 19:30:36 +0000
Subject: [PATCH 12/15] get rid of elementwise seq_cst tests
---
.../NVPTX/atomicrmw-elementwise-sm60.ll | 879 ++---------------
.../NVPTX/atomicrmw-elementwise-sm70.ll | 887 ++----------------
.../NVPTX/atomicrmw-elementwise-sm90.ll | 842 ++---------------
.../NVPTX/expand-atomic-rmw-elementwise.ll | 20 -
4 files changed, 236 insertions(+), 2392 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index 4a7b2132cfb3c..bf5d392ff0b9b 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -16888,239 +16888,9 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
-define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v1i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<16>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
-; SM60-NEXT: mov.b32 %r9, 255;
-; SM60-NEXT: shl.b32 %r2, %r9, %r1;
-; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
-; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
-; SM60-NEXT: $L__BB348_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: add.s32 %r10, %r15, %r4;
-; SM60-NEXT: and.b32 %r11, %r10, %r2;
-; SM60-NEXT: and.b32 %r12, %r15, %r3;
-; SM60-NEXT: or.b32 %r13, %r12, %r11;
-; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
-; SM60-NEXT: setp.ne.b32 %p1, %r5, %r15;
-; SM60-NEXT: mov.b32 %r15, %r5;
-; SM60-NEXT: @%p1 bra $L__BB348_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r14, %r5, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b8 [func_retval0], %r14;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v2i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<12>;
-; SM60-NEXT: .reg .b32 %r<16>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r15, [%rd1];
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM60-NEXT: $L__BB349_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r15, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM60-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM60-NEXT: and.b16 %rs9, %rs8, 255;
-; SM60-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM60-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM60-NEXT: shl.b32 %r11, %r10, %r1;
-; SM60-NEXT: and.b32 %r12, %r15, %r2;
-; SM60-NEXT: or.b32 %r13, %r12, %r11;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM60-NEXT: mov.b32 %r15, %r3;
-; SM60-NEXT: @%p1 bra $L__BB349_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r14, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r14;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
- ret <2 x i8> %retval
-}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v4i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<13>;
-; SM60-NEXT: .reg .b32 %r<19>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: $L__BB350_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM60-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM60-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM60-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM60-NEXT: mov.b32 %r18, %r1;
-; SM60-NEXT: @%p1 bra $L__BB350_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: add_seq_cst_v8i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<25>;
-; SM60-NEXT: .reg .b32 %r<35>;
-; SM60-NEXT: .reg .b64 %rd<12>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM60-NEXT: $L__BB351_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM60-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM60-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM60-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM60-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM60-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM60-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM60-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM60-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM60-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM60-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM60-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM60-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM60-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM60-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB351_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM60-LABEL: add_monotonic_v1i32_global_cta(
@@ -17361,90 +17131,9 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
-define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v1i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v2i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<5>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v4i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM60-LABEL: add_seq_cst_v8i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<17>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM60-NEXT: atom.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM60-NEXT: atom.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM60-NEXT: atom.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM60-NEXT: atom.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM60-NEXT: atom.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM60-NEXT: atom.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM60-NEXT: atom.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM60-LABEL: nand_monotonic_v1i8_global_cta(
@@ -17465,7 +17154,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM60-NEXT: not.b32 %r3, %r2;
; SM60-NEXT: shl.b32 %r4, %r6, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB360_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r10, %r16, %r4;
; SM60-NEXT: not.b32 %r11, %r10;
@@ -17475,7 +17164,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM60-NEXT: mov.b32 %r16, %r5;
-; SM60-NEXT: @%p1 bra $L__BB368_1;
+; SM60-NEXT: @%p1 bra $L__BB360_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: st.param.b8 [func_retval0], %r15;
@@ -17504,7 +17193,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB361_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17522,7 +17211,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB369_1;
+; SM60-NEXT: @%p1 bra $L__BB361_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: st.param.b16 [func_retval0], %r17;
@@ -17542,14 +17231,14 @@ define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM60-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB362_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB370_1;
+; SM60-NEXT: @%p1 bra $L__BB362_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
@@ -17570,7 +17259,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB363_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -17588,7 +17277,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB371_1;
+; SM60-NEXT: @%p1 bra $L__BB363_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM60-NEXT: ret;
@@ -17615,7 +17304,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: not.b32 %r3, %r2;
; SM60-NEXT: shl.b32 %r4, %r6, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB364_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r10, %r16, %r4;
; SM60-NEXT: not.b32 %r11, %r10;
@@ -17625,7 +17314,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM60-NEXT: mov.b32 %r16, %r5;
-; SM60-NEXT: @%p1 bra $L__BB372_1;
+; SM60-NEXT: @%p1 bra $L__BB364_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: membar.cta;
@@ -17655,7 +17344,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB365_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17673,7 +17362,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB373_1;
+; SM60-NEXT: @%p1 bra $L__BB365_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
; SM60-NEXT: membar.cta;
@@ -17694,14 +17383,14 @@ define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB366_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB374_1;
+; SM60-NEXT: @%p1 bra $L__BB366_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
@@ -17723,7 +17412,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB367_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -17741,7 +17430,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB375_1;
+; SM60-NEXT: @%p1 bra $L__BB367_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -17770,7 +17459,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: not.b32 %r3, %r2;
; SM60-NEXT: shl.b32 %r4, %r6, %r1;
; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB368_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r10, %r16, %r4;
; SM60-NEXT: not.b32 %r11, %r10;
@@ -17780,7 +17469,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM60-NEXT: mov.b32 %r16, %r5;
-; SM60-NEXT: @%p1 bra $L__BB376_1;
+; SM60-NEXT: @%p1 bra $L__BB368_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: st.param.b8 [func_retval0], %r15;
@@ -17800,162 +17489,7 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB377_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB377_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
-}
-
-define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: nand_release_v4i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<6>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB378_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r2;
-; SM60-NEXT: xor.b32 %r4, %r3, -1;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB378_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
-}
-
-define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_release_v8i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<12>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB379_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r8, %r2;
-; SM60-NEXT: and.b32 %r4, %r7, %r1;
-; SM60-NEXT: xor.b32 %r5, %r4, -1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM60-NEXT: xor.b32 %r6, %r3, -1;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB379_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
-}
-
-define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v1i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<17>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: and.b32 %r8, %r7, 3;
-; SM60-NEXT: shl.b32 %r1, %r8, 3;
-; SM60-NEXT: mov.b32 %r9, 255;
-; SM60-NEXT: shl.b32 %r2, %r9, %r1;
-; SM60-NEXT: not.b32 %r3, %r2;
-; SM60-NEXT: shl.b32 %r4, %r6, %r1;
-; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
-; SM60-NEXT: $L__BB380_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r10, %r16, %r4;
-; SM60-NEXT: not.b32 %r11, %r10;
-; SM60-NEXT: and.b32 %r12, %r11, %r2;
-; SM60-NEXT: and.b32 %r13, %r16, %r3;
-; SM60-NEXT: or.b32 %r14, %r13, %r12;
-; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
-; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
-; SM60-NEXT: mov.b32 %r16, %r5;
-; SM60-NEXT: @%p1 bra $L__BB380_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r15, %r5, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b8 [func_retval0], %r15;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v2i8_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM60-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -17965,7 +17499,7 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: shl.b32 %r8, %r7, %r1;
; SM60-NEXT: not.b32 %r2, %r8;
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
-; SM60-NEXT: $L__BB381_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB369_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: shr.u32 %r9, %r18, %r1;
; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17983,59 +17517,57 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM60-NEXT: mov.b32 %r18, %r3;
-; SM60-NEXT: @%p1 bra $L__BB381_1;
+; SM60-NEXT: @%p1 bra $L__BB369_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b16 [func_retval0], %r17;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
}
-define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v4i8_global_cta(
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM60-LABEL: nand_release_v4i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b32 %r<6>;
; SM60-NEXT: .reg .b64 %rd<2>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
+; SM60-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB382_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB370_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r2;
; SM60-NEXT: xor.b32 %r4, %r3, -1;
; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM60-NEXT: mov.b32 %r5, %r1;
-; SM60-NEXT: @%p1 bra $L__BB382_1;
+; SM60-NEXT: @%p1 bra $L__BB370_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r1;
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
}
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM60-LABEL: nand_seq_cst_v8i8_global_cta(
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM60-LABEL: nand_release_v8i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
; SM60-NEXT: .reg .b32 %r<9>;
; SM60-NEXT: .reg .b64 %rd<12>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
+; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM60-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM60-NEXT: $L__BB383_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB371_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r7;
; SM60-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -18053,15 +17585,18 @@ define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM60-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM60-NEXT: @%p1 bra $L__BB383_1;
+; SM60-NEXT: @%p1 bra $L__BB371_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
}
+
+
+
+
define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM60-LABEL: nand_monotonic_v1i32_global_cta(
; SM60: {
@@ -18073,14 +17608,14 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM60-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
; SM60-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB372_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r1;
; SM60-NEXT: not.b32 %r4, %r3;
; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM60-NEXT: mov.b32 %r5, %r2;
-; SM60-NEXT: @%p1 bra $L__BB384_1;
+; SM60-NEXT: @%p1 bra $L__BB372_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
@@ -18101,7 +17636,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB373_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18118,7 +17653,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB385_1;
+; SM60-NEXT: @%p1 bra $L__BB373_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-NEXT: ret;
@@ -18139,7 +17674,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB374_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18156,12 +17691,12 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB386_1;
+; SM60-NEXT: @%p1 bra $L__BB374_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB386_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB374_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18178,7 +17713,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB386_3;
+; SM60-NEXT: @%p2 bra $L__BB374_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
@@ -18200,7 +17735,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB375_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18217,12 +17752,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB387_1;
+; SM60-NEXT: @%p1 bra $L__BB375_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB387_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB375_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18239,12 +17774,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB387_3;
+; SM60-NEXT: @%p2 bra $L__BB375_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB387_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB375_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18261,12 +17796,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB387_5;
+; SM60-NEXT: @%p3 bra $L__BB375_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB387_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB375_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18283,7 +17818,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB387_7;
+; SM60-NEXT: @%p4 bra $L__BB375_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -18303,14 +17838,14 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
; SM60-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB376_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r1;
; SM60-NEXT: not.b32 %r4, %r3;
; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM60-NEXT: mov.b32 %r5, %r2;
-; SM60-NEXT: @%p1 bra $L__BB388_1;
+; SM60-NEXT: @%p1 bra $L__BB376_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
@@ -18332,7 +17867,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB377_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18349,7 +17884,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB389_1;
+; SM60-NEXT: @%p1 bra $L__BB377_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -18371,7 +17906,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB378_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18388,12 +17923,12 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB390_1;
+; SM60-NEXT: @%p1 bra $L__BB378_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB378_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18410,7 +17945,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB390_3;
+; SM60-NEXT: @%p2 bra $L__BB378_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -18433,7 +17968,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB379_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18450,12 +17985,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB391_1;
+; SM60-NEXT: @%p1 bra $L__BB379_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB391_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB379_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18472,12 +18007,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB391_3;
+; SM60-NEXT: @%p2 bra $L__BB379_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB391_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB379_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18494,12 +18029,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB391_5;
+; SM60-NEXT: @%p3 bra $L__BB379_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB391_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB379_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18516,7 +18051,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB391_7;
+; SM60-NEXT: @%p4 bra $L__BB379_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -18538,14 +18073,14 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB380_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: and.b32 %r3, %r5, %r1;
; SM60-NEXT: not.b32 %r4, %r3;
; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM60-NEXT: mov.b32 %r5, %r2;
-; SM60-NEXT: @%p1 bra $L__BB392_1;
+; SM60-NEXT: @%p1 bra $L__BB380_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
@@ -18567,7 +18102,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB381_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18584,7 +18119,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB393_1;
+; SM60-NEXT: @%p1 bra $L__BB381_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM60-NEXT: ret;
@@ -18606,7 +18141,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB382_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18623,12 +18158,12 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB394_1;
+; SM60-NEXT: @%p1 bra $L__BB382_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM60-NEXT: $L__BB382_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18645,7 +18180,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB394_3;
+; SM60-NEXT: @%p2 bra $L__BB382_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end1
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM60-NEXT: ret;
@@ -18668,7 +18203,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM60-NEXT: $L__BB383_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18685,12 +18220,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB395_1;
+; SM60-NEXT: @%p1 bra $L__BB383_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB395_3: // %atomicrmw.start6
+; SM60-NEXT: $L__BB383_3: // %atomicrmw.start6
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18707,12 +18242,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB395_3;
+; SM60-NEXT: @%p2 bra $L__BB383_3;
; SM60-NEXT: // %bb.4: // %atomicrmw.end5
; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB395_5: // %atomicrmw.start16
+; SM60-NEXT: $L__BB383_5: // %atomicrmw.start16
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18729,12 +18264,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB395_5;
+; SM60-NEXT: @%p3 bra $L__BB383_5;
; SM60-NEXT: // %bb.6: // %atomicrmw.end15
; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB395_7: // %atomicrmw.start22
+; SM60-NEXT: $L__BB383_7: // %atomicrmw.start22
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18751,7 +18286,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB395_7;
+; SM60-NEXT: @%p4 bra $L__BB383_7;
; SM60-NEXT: // %bb.8: // %atomicrmw.end21
; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -18759,241 +18294,3 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
}
-
-define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v1i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<6>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b32 %r5, [%rd1];
-; SM60-NEXT: $L__BB396_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: and.b32 %r3, %r5, %r1;
-; SM60-NEXT: not.b32 %r4, %r3;
-; SM60-NEXT: atom.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM60-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM60-NEXT: mov.b32 %r5, %r2;
-; SM60-NEXT: @%p1 bra $L__BB396_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v2i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<7>;
-; SM60-NEXT: .reg .b64 %rd<13>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: $L__BB397_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r3, %r6, %r2;
-; SM60-NEXT: and.b32 %r4, %r5, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB397_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v4i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b32 %r<13>;
-; SM60-NEXT: .reg .b64 %rd<24>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT: $L__BB398_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r10;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r5, %r10, %r2;
-; SM60-NEXT: and.b32 %r6, %r9, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r5;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB398_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM60-NEXT: $L__BB398_3: // %atomicrmw.start2
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM60-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM60-NEXT: and.b32 %r7, %r12, %r4;
-; SM60-NEXT: and.b32 %r8, %r11, %r3;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM60-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM60-NEXT: not.b64 %rd22, %rd21;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM60-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB398_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end1
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-
-define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM60-LABEL: nand_seq_cst_v8i32_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<5>;
-; SM60-NEXT: .reg .b32 %r<25>;
-; SM60-NEXT: .reg .b64 %rd<46>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
-; SM60-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
-; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
-; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM60-NEXT: $L__BB399_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM60-NEXT: cvt.u64.u32 %rd4, %r18;
-; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM60-NEXT: and.b32 %r9, %r18, %r6;
-; SM60-NEXT: and.b32 %r10, %r17, %r5;
-; SM60-NEXT: cvt.u64.u32 %rd7, %r10;
-; SM60-NEXT: cvt.u64.u32 %rd8, %r9;
-; SM60-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM60-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM60-NEXT: not.b64 %rd11, %rd10;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM60-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
-; SM60-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM60-NEXT: @%p1 bra $L__BB399_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: ld.volatile.global.b64 %rd13, [%rd1+8];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
-; SM60-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM60-NEXT: $L__BB399_3: // %atomicrmw.start6
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd14, %r19;
-; SM60-NEXT: cvt.u64.u32 %rd15, %r20;
-; SM60-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM60-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM60-NEXT: and.b32 %r11, %r20, %r8;
-; SM60-NEXT: and.b32 %r12, %r19, %r7;
-; SM60-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM60-NEXT: cvt.u64.u32 %rd19, %r11;
-; SM60-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM60-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM60-NEXT: not.b64 %rd22, %rd21;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM60-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
-; SM60-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM60-NEXT: @%p2 bra $L__BB399_3;
-; SM60-NEXT: // %bb.4: // %atomicrmw.end5
-; SM60-NEXT: ld.volatile.global.b64 %rd24, [%rd1+16];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
-; SM60-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM60-NEXT: $L__BB399_5: // %atomicrmw.start16
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd25, %r21;
-; SM60-NEXT: cvt.u64.u32 %rd26, %r22;
-; SM60-NEXT: shl.b64 %rd27, %rd26, 32;
-; SM60-NEXT: or.b64 %rd28, %rd25, %rd27;
-; SM60-NEXT: and.b32 %r13, %r22, %r2;
-; SM60-NEXT: and.b32 %r14, %r21, %r1;
-; SM60-NEXT: cvt.u64.u32 %rd29, %r14;
-; SM60-NEXT: cvt.u64.u32 %rd30, %r13;
-; SM60-NEXT: shl.b64 %rd31, %rd30, 32;
-; SM60-NEXT: or.b64 %rd32, %rd29, %rd31;
-; SM60-NEXT: not.b64 %rd33, %rd32;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
-; SM60-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
-; SM60-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM60-NEXT: @%p3 bra $L__BB399_5;
-; SM60-NEXT: // %bb.6: // %atomicrmw.end15
-; SM60-NEXT: ld.volatile.global.b64 %rd35, [%rd1+24];
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
-; SM60-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM60-NEXT: $L__BB399_7: // %atomicrmw.start22
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: cvt.u64.u32 %rd36, %r23;
-; SM60-NEXT: cvt.u64.u32 %rd37, %r24;
-; SM60-NEXT: shl.b64 %rd38, %rd37, 32;
-; SM60-NEXT: or.b64 %rd39, %rd36, %rd38;
-; SM60-NEXT: and.b32 %r15, %r24, %r4;
-; SM60-NEXT: and.b32 %r16, %r23, %r3;
-; SM60-NEXT: cvt.u64.u32 %rd40, %r16;
-; SM60-NEXT: cvt.u64.u32 %rd41, %r15;
-; SM60-NEXT: shl.b64 %rd42, %rd41, 32;
-; SM60-NEXT: or.b64 %rd43, %rd40, %rd42;
-; SM60-NEXT: not.b64 %rd44, %rd43;
-; SM60-NEXT: atom.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
-; SM60-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
-; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
-; SM60-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM60-NEXT: @%p4 bra $L__BB399_7;
-; SM60-NEXT: // %bb.8: // %atomicrmw.end21
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM60-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
-; SM60-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index a2a37ec0bb265..32152afa7c416 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -16812,239 +16812,9 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
-define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v1i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<16>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
-; SM70-NEXT: mov.b32 %r9, 255;
-; SM70-NEXT: shl.b32 %r2, %r9, %r1;
-; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM70-NEXT: $L__BB348_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: add.s32 %r10, %r15, %r4;
-; SM70-NEXT: and.b32 %r11, %r10, %r2;
-; SM70-NEXT: and.b32 %r12, %r15, %r3;
-; SM70-NEXT: or.b32 %r13, %r12, %r11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
-; SM70-NEXT: setp.ne.b32 %p1, %r5, %r15;
-; SM70-NEXT: mov.b32 %r15, %r5;
-; SM70-NEXT: @%p1 bra $L__BB348_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r14, %r5, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b8 [func_retval0], %r14;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v2i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<12>;
-; SM70-NEXT: .reg .b32 %r<16>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT: $L__BB349_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r15, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: add.s16 %rs7, %rs4, %rs6;
-; SM70-NEXT: add.s16 %rs8, %rs3, %rs5;
-; SM70-NEXT: and.b16 %rs9, %rs8, 255;
-; SM70-NEXT: shl.b16 %rs10, %rs7, 8;
-; SM70-NEXT: or.b16 %rs11, %rs9, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r10, %rs11;
-; SM70-NEXT: shl.b32 %r11, %r10, %r1;
-; SM70-NEXT: and.b32 %r12, %r15, %r2;
-; SM70-NEXT: or.b32 %r13, %r12, %r11;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r15;
-; SM70-NEXT: mov.b32 %r15, %r3;
-; SM70-NEXT: @%p1 bra $L__BB349_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r14, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r14;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
- ret <2 x i8> %retval
-}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<13>;
-; SM70-NEXT: .reg .b32 %r<19>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: $L__BB350_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM70-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM70-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM70-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM70-NEXT: mov.b32 %r18, %r1;
-; SM70-NEXT: @%p1 bra $L__BB350_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: add_seq_cst_v8i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<25>;
-; SM70-NEXT: .reg .b32 %r<35>;
-; SM70-NEXT: .reg .b64 %rd<12>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM70-NEXT: $L__BB351_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM70-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM70-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM70-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM70-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM70-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM70-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM70-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM70-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM70-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM70-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM70-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM70-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM70-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM70-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB351_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: add_monotonic_v1i32_global_cta(
@@ -17285,90 +17055,9 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
-define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v1i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v2i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<5>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v4i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM70-LABEL: add_seq_cst_v8i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<17>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM70-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: nand_monotonic_v1i8_global_cta(
@@ -17389,7 +17078,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM70-NEXT: not.b32 %r3, %r2;
; SM70-NEXT: shl.b32 %r4, %r6, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB360_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r10, %r16, %r4;
; SM70-NEXT: not.b32 %r11, %r10;
@@ -17399,7 +17088,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM70-NEXT: mov.b32 %r16, %r5;
-; SM70-NEXT: @%p1 bra $L__BB368_1;
+; SM70-NEXT: @%p1 bra $L__BB360_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: st.param.b8 [func_retval0], %r15;
@@ -17428,7 +17117,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB361_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17446,7 +17135,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB369_1;
+; SM70-NEXT: @%p1 bra $L__BB361_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: st.param.b16 [func_retval0], %r17;
@@ -17466,14 +17155,14 @@ define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM70-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB362_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB370_1;
+; SM70-NEXT: @%p1 bra $L__BB362_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
@@ -17494,7 +17183,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB363_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -17512,7 +17201,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB371_1;
+; SM70-NEXT: @%p1 bra $L__BB363_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT: ret;
@@ -17539,7 +17228,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: not.b32 %r3, %r2;
; SM70-NEXT: shl.b32 %r4, %r6, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB364_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r10, %r16, %r4;
; SM70-NEXT: not.b32 %r11, %r10;
@@ -17549,7 +17238,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM70-NEXT: mov.b32 %r16, %r5;
-; SM70-NEXT: @%p1 bra $L__BB372_1;
+; SM70-NEXT: @%p1 bra $L__BB364_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -17579,7 +17268,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB365_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17597,7 +17286,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB373_1;
+; SM70-NEXT: @%p1 bra $L__BB365_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
; SM70-NEXT: fence.acq_rel.cta;
@@ -17618,14 +17307,14 @@ define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB366_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB374_1;
+; SM70-NEXT: @%p1 bra $L__BB366_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
@@ -17647,7 +17336,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB367_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -17665,7 +17354,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB375_1;
+; SM70-NEXT: @%p1 bra $L__BB367_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -17694,7 +17383,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: not.b32 %r3, %r2;
; SM70-NEXT: shl.b32 %r4, %r6, %r1;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB368_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r10, %r16, %r4;
; SM70-NEXT: not.b32 %r11, %r10;
@@ -17704,7 +17393,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM70-NEXT: mov.b32 %r16, %r5;
-; SM70-NEXT: @%p1 bra $L__BB376_1;
+; SM70-NEXT: @%p1 bra $L__BB368_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: st.param.b8 [func_retval0], %r15;
@@ -17722,165 +17411,10 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB377_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB377_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
- ret <2 x i8> %retval
-}
-
-define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: nand_release_v4i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<6>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB378_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r2;
-; SM70-NEXT: xor.b32 %r4, %r3, -1;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB378_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
- ret <4 x i8> %retval
-}
-
-define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_release_v8i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<12>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB379_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r8, %r2;
-; SM70-NEXT: and.b32 %r4, %r7, %r1;
-; SM70-NEXT: xor.b32 %r5, %r4, -1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM70-NEXT: xor.b32 %r6, %r3, -1;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
-; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB379_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
- ret <8 x i8> %retval
-}
-
-define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v1i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<17>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: and.b32 %r8, %r7, 3;
-; SM70-NEXT: shl.b32 %r1, %r8, 3;
-; SM70-NEXT: mov.b32 %r9, 255;
-; SM70-NEXT: shl.b32 %r2, %r9, %r1;
-; SM70-NEXT: not.b32 %r3, %r2;
-; SM70-NEXT: shl.b32 %r4, %r6, %r1;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM70-NEXT: $L__BB380_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r10, %r16, %r4;
-; SM70-NEXT: not.b32 %r11, %r10;
-; SM70-NEXT: and.b32 %r12, %r11, %r2;
-; SM70-NEXT: and.b32 %r13, %r16, %r3;
-; SM70-NEXT: or.b32 %r14, %r13, %r12;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
-; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
-; SM70-NEXT: mov.b32 %r16, %r5;
-; SM70-NEXT: @%p1 bra $L__BB380_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r15, %r5, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b8 [func_retval0], %r15;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v2i8_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
+; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM70-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
; SM70-NEXT: and.b32 %r6, %r5, 3;
@@ -17889,7 +17423,7 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: shl.b32 %r8, %r7, %r1;
; SM70-NEXT: not.b32 %r2, %r8;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM70-NEXT: $L__BB381_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB369_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: shr.u32 %r9, %r18, %r1;
; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -17907,59 +17441,57 @@ define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM70-NEXT: mov.b32 %r18, %r3;
-; SM70-NEXT: @%p1 bra $L__BB381_1;
+; SM70-NEXT: @%p1 bra $L__BB369_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b16 [func_retval0], %r17;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
}
-define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v4i8_global_cta(
+define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
+; SM70-LABEL: nand_release_v4i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b32 %r<6>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB382_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB370_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r2;
; SM70-NEXT: xor.b32 %r4, %r3, -1;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT: mov.b32 %r5, %r1;
-; SM70-NEXT: @%p1 bra $L__BB382_1;
+; SM70-NEXT: @%p1 bra $L__BB370_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r1;
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
}
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM70-LABEL: nand_seq_cst_v8i8_global_cta(
+define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
+; SM70-LABEL: nand_release_v8i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
+; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM70-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM70-NEXT: $L__BB383_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB371_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r7;
; SM70-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -17977,15 +17509,18 @@ define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM70-NEXT: @%p1 bra $L__BB383_1;
+; SM70-NEXT: @%p1 bra $L__BB371_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
+ %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
}
+
+
+
+
define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: nand_monotonic_v1i32_global_cta(
; SM70: {
@@ -17997,14 +17532,14 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM70-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
; SM70-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB372_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r1;
; SM70-NEXT: not.b32 %r4, %r3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT: mov.b32 %r5, %r2;
-; SM70-NEXT: @%p1 bra $L__BB384_1;
+; SM70-NEXT: @%p1 bra $L__BB372_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
@@ -18025,7 +17560,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB373_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18042,7 +17577,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB385_1;
+; SM70-NEXT: @%p1 bra $L__BB373_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT: ret;
@@ -18063,7 +17598,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB374_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18080,12 +17615,12 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB386_1;
+; SM70-NEXT: @%p1 bra $L__BB374_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB386_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB374_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18102,7 +17637,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB386_3;
+; SM70-NEXT: @%p2 bra $L__BB374_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
@@ -18124,7 +17659,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB375_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18141,12 +17676,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB387_1;
+; SM70-NEXT: @%p1 bra $L__BB375_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB387_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB375_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18163,12 +17698,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB387_3;
+; SM70-NEXT: @%p2 bra $L__BB375_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB387_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB375_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18185,12 +17720,12 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB387_5;
+; SM70-NEXT: @%p3 bra $L__BB375_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB387_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB375_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18207,7 +17742,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB387_7;
+; SM70-NEXT: @%p4 bra $L__BB375_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -18227,14 +17762,14 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
; SM70-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB376_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r1;
; SM70-NEXT: not.b32 %r4, %r3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT: mov.b32 %r5, %r2;
-; SM70-NEXT: @%p1 bra $L__BB388_1;
+; SM70-NEXT: @%p1 bra $L__BB376_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
@@ -18256,7 +17791,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB377_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18273,7 +17808,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB389_1;
+; SM70-NEXT: @%p1 bra $L__BB377_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -18295,7 +17830,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB378_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18312,12 +17847,12 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB390_1;
+; SM70-NEXT: @%p1 bra $L__BB378_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB390_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB378_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18334,7 +17869,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB390_3;
+; SM70-NEXT: @%p2 bra $L__BB378_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -18357,7 +17892,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB379_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18374,12 +17909,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB391_1;
+; SM70-NEXT: @%p1 bra $L__BB379_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB391_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB379_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18396,12 +17931,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB391_3;
+; SM70-NEXT: @%p2 bra $L__BB379_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB391_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB379_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18418,12 +17953,12 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB391_5;
+; SM70-NEXT: @%p3 bra $L__BB379_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB391_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB379_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18440,7 +17975,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB391_7;
+; SM70-NEXT: @%p4 bra $L__BB379_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -18462,14 +17997,14 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB380_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: and.b32 %r3, %r5, %r1;
; SM70-NEXT: not.b32 %r4, %r3;
; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT: mov.b32 %r5, %r2;
-; SM70-NEXT: @%p1 bra $L__BB392_1;
+; SM70-NEXT: @%p1 bra $L__BB380_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.b32 [func_retval0], %r2;
; SM70-NEXT: ret;
@@ -18491,7 +18026,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB381_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -18508,7 +18043,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB393_1;
+; SM70-NEXT: @%p1 bra $L__BB381_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT: ret;
@@ -18530,7 +18065,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB382_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
@@ -18547,12 +18082,12 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB394_1;
+; SM70-NEXT: @%p1 bra $L__BB382_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB394_3: // %atomicrmw.start2
+; SM70-NEXT: $L__BB382_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
@@ -18569,7 +18104,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB394_3;
+; SM70-NEXT: @%p2 bra $L__BB382_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end1
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT: ret;
@@ -18592,7 +18127,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM70-NEXT: $L__BB383_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
@@ -18609,12 +18144,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB395_1;
+; SM70-NEXT: @%p1 bra $L__BB383_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB395_3: // %atomicrmw.start6
+; SM70-NEXT: $L__BB383_3: // %atomicrmw.start6
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
@@ -18631,12 +18166,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB395_3;
+; SM70-NEXT: @%p2 bra $L__BB383_3;
; SM70-NEXT: // %bb.4: // %atomicrmw.end5
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB395_5: // %atomicrmw.start16
+; SM70-NEXT: $L__BB383_5: // %atomicrmw.start16
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
@@ -18653,12 +18188,12 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB395_5;
+; SM70-NEXT: @%p3 bra $L__BB383_5;
; SM70-NEXT: // %bb.6: // %atomicrmw.end15
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB395_7: // %atomicrmw.start22
+; SM70-NEXT: $L__BB383_7: // %atomicrmw.start22
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
@@ -18675,7 +18210,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB395_7;
+; SM70-NEXT: @%p4 bra $L__BB383_7;
; SM70-NEXT: // %bb.8: // %atomicrmw.end21
; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -18683,241 +18218,3 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
}
-
-define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v1i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<6>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM70-NEXT: $L__BB396_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: and.b32 %r3, %r5, %r1;
-; SM70-NEXT: not.b32 %r4, %r3;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM70-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM70-NEXT: mov.b32 %r5, %r2;
-; SM70-NEXT: @%p1 bra $L__BB396_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v2i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<7>;
-; SM70-NEXT: .reg .b64 %rd<13>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: $L__BB397_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r3, %r6, %r2;
-; SM70-NEXT: and.b32 %r4, %r5, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB397_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v4i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b32 %r<13>;
-; SM70-NEXT: .reg .b64 %rd<24>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT: $L__BB398_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r9;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r10;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r5, %r10, %r2;
-; SM70-NEXT: and.b32 %r6, %r9, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r6;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r5;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r9, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB398_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd13;
-; SM70-NEXT: $L__BB398_3: // %atomicrmw.start2
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM70-NEXT: cvt.u64.u32 %rd15, %r12;
-; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM70-NEXT: and.b32 %r7, %r12, %r4;
-; SM70-NEXT: and.b32 %r8, %r11, %r3;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r8;
-; SM70-NEXT: cvt.u64.u32 %rd19, %r7;
-; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM70-NEXT: not.b64 %rd22, %rd21;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
-; SM70-NEXT: cvt.u32.u64 %r11, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB398_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end1
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-
-define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM70-LABEL: nand_seq_cst_v8i32_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<5>;
-; SM70-NEXT: .reg .b32 %r<25>;
-; SM70-NEXT: .reg .b64 %rd<46>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
-; SM70-NEXT: fence.sc.cta;
-; SM70-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1+16];
-; SM70-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1];
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
-; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
-; SM70-NEXT: $L__BB399_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM70-NEXT: cvt.u64.u32 %rd4, %r18;
-; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM70-NEXT: and.b32 %r9, %r18, %r6;
-; SM70-NEXT: and.b32 %r10, %r17, %r5;
-; SM70-NEXT: cvt.u64.u32 %rd7, %r10;
-; SM70-NEXT: cvt.u64.u32 %rd8, %r9;
-; SM70-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM70-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM70-NEXT: not.b64 %rd11, %rd10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM70-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
-; SM70-NEXT: cvt.u32.u64 %r17, %rd12;
-; SM70-NEXT: @%p1 bra $L__BB399_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
-; SM70-NEXT: cvt.u32.u64 %r19, %rd13;
-; SM70-NEXT: $L__BB399_3: // %atomicrmw.start6
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd14, %r19;
-; SM70-NEXT: cvt.u64.u32 %rd15, %r20;
-; SM70-NEXT: shl.b64 %rd16, %rd15, 32;
-; SM70-NEXT: or.b64 %rd17, %rd14, %rd16;
-; SM70-NEXT: and.b32 %r11, %r20, %r8;
-; SM70-NEXT: and.b32 %r12, %r19, %r7;
-; SM70-NEXT: cvt.u64.u32 %rd18, %r12;
-; SM70-NEXT: cvt.u64.u32 %rd19, %r11;
-; SM70-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM70-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM70-NEXT: not.b64 %rd22, %rd21;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
-; SM70-NEXT: setp.ne.b64 %p2, %rd23, %rd17;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
-; SM70-NEXT: cvt.u32.u64 %r19, %rd23;
-; SM70-NEXT: @%p2 bra $L__BB399_3;
-; SM70-NEXT: // %bb.4: // %atomicrmw.end5
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
-; SM70-NEXT: cvt.u32.u64 %r21, %rd24;
-; SM70-NEXT: $L__BB399_5: // %atomicrmw.start16
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd25, %r21;
-; SM70-NEXT: cvt.u64.u32 %rd26, %r22;
-; SM70-NEXT: shl.b64 %rd27, %rd26, 32;
-; SM70-NEXT: or.b64 %rd28, %rd25, %rd27;
-; SM70-NEXT: and.b32 %r13, %r22, %r2;
-; SM70-NEXT: and.b32 %r14, %r21, %r1;
-; SM70-NEXT: cvt.u64.u32 %rd29, %r14;
-; SM70-NEXT: cvt.u64.u32 %rd30, %r13;
-; SM70-NEXT: shl.b64 %rd31, %rd30, 32;
-; SM70-NEXT: or.b64 %rd32, %rd29, %rd31;
-; SM70-NEXT: not.b64 %rd33, %rd32;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
-; SM70-NEXT: setp.ne.b64 %p3, %rd34, %rd28;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
-; SM70-NEXT: cvt.u32.u64 %r21, %rd34;
-; SM70-NEXT: @%p3 bra $L__BB399_5;
-; SM70-NEXT: // %bb.6: // %atomicrmw.end15
-; SM70-NEXT: ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
-; SM70-NEXT: cvt.u32.u64 %r23, %rd35;
-; SM70-NEXT: $L__BB399_7: // %atomicrmw.start22
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: cvt.u64.u32 %rd36, %r23;
-; SM70-NEXT: cvt.u64.u32 %rd37, %r24;
-; SM70-NEXT: shl.b64 %rd38, %rd37, 32;
-; SM70-NEXT: or.b64 %rd39, %rd36, %rd38;
-; SM70-NEXT: and.b32 %r15, %r24, %r4;
-; SM70-NEXT: and.b32 %r16, %r23, %r3;
-; SM70-NEXT: cvt.u64.u32 %rd40, %r16;
-; SM70-NEXT: cvt.u64.u32 %rd41, %r15;
-; SM70-NEXT: shl.b64 %rd42, %rd41, 32;
-; SM70-NEXT: or.b64 %rd43, %rd40, %rd42;
-; SM70-NEXT: not.b64 %rd44, %rd43;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
-; SM70-NEXT: setp.ne.b64 %p4, %rd45, %rd39;
-; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
-; SM70-NEXT: cvt.u32.u64 %r23, %rd45;
-; SM70-NEXT: @%p4 bra $L__BB399_7;
-; SM70-NEXT: // %bb.8: // %atomicrmw.end21
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM70-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
-; SM70-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index a7ea56b5719aa..2e07eaadd297a 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -15881,239 +15881,9 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
ret <8 x i8> %retval
}
-define <1 x i8> @add_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v1i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<16>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v1i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.b8 %r6, [add_seq_cst_v1i8_global_cta_param_1];
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
-; SM90-NEXT: mov.b32 %r9, 255;
-; SM90-NEXT: shl.b32 %r2, %r9, %r1;
-; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1];
-; SM90-NEXT: $L__BB348_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: add.s32 %r10, %r15, %r4;
-; SM90-NEXT: and.b32 %r11, %r10, %r2;
-; SM90-NEXT: and.b32 %r12, %r15, %r3;
-; SM90-NEXT: or.b32 %r13, %r12, %r11;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
-; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15;
-; SM90-NEXT: mov.b32 %r15, %r5;
-; SM90-NEXT: @%p1 bra $L__BB348_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r14, %r5, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-define <2 x i8> @add_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v2i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<10>;
-; SM90-NEXT: .reg .b32 %r<18>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_seq_cst_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM90-NEXT: $L__BB349_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r17, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: add.s16x2 %r11, %r10, %r4;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r11;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: and.b16 %rs8, %rs5, 255;
-; SM90-NEXT: or.b16 %rs9, %rs8, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: shl.b32 %r13, %r12, %r1;
-; SM90-NEXT: and.b32 %r14, %r17, %r2;
-; SM90-NEXT: or.b32 %r15, %r14, %r13;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM90-NEXT: mov.b32 %r17, %r3;
-; SM90-NEXT: @%p1 bra $L__BB349_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r16, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
- ret <2 x i8> %retval
-}
-define <4 x i8> @add_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v4i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<13>;
-; SM90-NEXT: .reg .b32 %r<19>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_seq_cst_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r2, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: $L__BB350_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r18, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r18, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r18, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r18;
-; SM90-NEXT: mov.b32 %r18, %r1;
-; SM90-NEXT: @%p1 bra $L__BB350_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-define <8 x i8> @add_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: add_seq_cst_v8i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<25>;
-; SM90-NEXT: .reg .b32 %r<35>;
-; SM90-NEXT: .reg .b64 %rd<12>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
-; SM90-NEXT: $L__BB351_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
-; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
-; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
-; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
-; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
-; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
-; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
-; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
-; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
-; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
-; SM90-NEXT: cvt.u32.u16 %r15, %rs12;
-; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
-; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
-; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
-; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
-; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
-; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
-; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
-; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
-; SM90-NEXT: add.s16 %rs18, %rs17, %rs16;
-; SM90-NEXT: cvt.u32.u16 %r23, %rs18;
-; SM90-NEXT: prmt.b32 %r24, %r23, %r20, 0x3340U;
-; SM90-NEXT: prmt.b32 %r25, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs19, %r25;
-; SM90-NEXT: prmt.b32 %r26, %r34, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs20, %r26;
-; SM90-NEXT: add.s16 %rs21, %rs20, %rs19;
-; SM90-NEXT: cvt.u32.u16 %r27, %rs21;
-; SM90-NEXT: prmt.b32 %r28, %r2, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs22, %r28;
-; SM90-NEXT: prmt.b32 %r29, %r34, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs23, %r29;
-; SM90-NEXT: add.s16 %rs24, %rs23, %rs22;
-; SM90-NEXT: cvt.u32.u16 %r30, %rs24;
-; SM90-NEXT: prmt.b32 %r31, %r30, %r27, 0x3340U;
-; SM90-NEXT: prmt.b32 %r32, %r31, %r24, 0x5410U;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r32;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r33;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r34;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd10;
-; SM90-NEXT: mov.b64 {_, %r34}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB351_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM90-LABEL: add_monotonic_v1i32_global_cta(
@@ -16354,90 +16124,9 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
ret <8 x i32> %retval
}
-define <1 x i32> @add_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v1i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.b32 %r1, [add_seq_cst_v1i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-define <2 x i32> @add_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_seq_cst_v2i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-define <4 x i32> @add_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v4i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v4i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-define <8 x i32> @add_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM90-LABEL: add_seq_cst_v8i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<17>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_v8i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_seq_cst_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_seq_cst_v8i32_global_cta_param_1];
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
-; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM90-LABEL: nand_monotonic_v1i8_global_cta(
@@ -16458,7 +16147,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: not.b32 %r3, %r2;
; SM90-NEXT: shl.b32 %r4, %r6, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM90-NEXT: $L__BB368_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB360_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r10, %r16, %r4;
; SM90-NEXT: not.b32 %r11, %r10;
@@ -16468,7 +16157,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM90-NEXT: mov.b32 %r16, %r5;
-; SM90-NEXT: @%p1 bra $L__BB368_1;
+; SM90-NEXT: @%p1 bra $L__BB360_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: st.param.b8 [func_retval0], %r15;
@@ -16497,7 +16186,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB369_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB361_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -16515,7 +16204,7 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB369_1;
+; SM90-NEXT: @%p1 bra $L__BB361_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: st.param.b16 [func_retval0], %r17;
@@ -16535,14 +16224,14 @@ define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB370_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB362_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB370_1;
+; SM90-NEXT: @%p1 bra $L__BB362_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
@@ -16563,7 +16252,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB371_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB363_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -16581,7 +16270,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB371_1;
+; SM90-NEXT: @%p1 bra $L__BB363_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
@@ -16608,7 +16297,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: not.b32 %r3, %r2;
; SM90-NEXT: shl.b32 %r4, %r6, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB364_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r10, %r16, %r4;
; SM90-NEXT: not.b32 %r11, %r10;
@@ -16618,7 +16307,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM90-NEXT: mov.b32 %r16, %r5;
-; SM90-NEXT: @%p1 bra $L__BB372_1;
+; SM90-NEXT: @%p1 bra $L__BB364_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -16648,7 +16337,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB373_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB365_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -16666,7 +16355,7 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB373_1;
+; SM90-NEXT: @%p1 bra $L__BB365_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
@@ -16687,14 +16376,14 @@ define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB374_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB366_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB374_1;
+; SM90-NEXT: @%p1 bra $L__BB366_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
@@ -16716,7 +16405,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB375_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB367_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -16734,7 +16423,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB375_1;
+; SM90-NEXT: @%p1 bra $L__BB367_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
@@ -16763,7 +16452,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: not.b32 %r3, %r2;
; SM90-NEXT: shl.b32 %r4, %r6, %r1;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB368_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r10, %r16, %r4;
; SM90-NEXT: not.b32 %r11, %r10;
@@ -16773,7 +16462,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
; SM90-NEXT: mov.b32 %r16, %r5;
-; SM90-NEXT: @%p1 bra $L__BB376_1;
+; SM90-NEXT: @%p1 bra $L__BB368_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: st.param.b8 [func_retval0], %r15;
@@ -16803,7 +16492,7 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: shl.b32 %r8, %r7, %r1;
; SM90-NEXT: not.b32 %r2, %r8;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB377_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB369_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: shr.u32 %r9, %r18, %r1;
; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
@@ -16821,7 +16510,7 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB377_1;
+; SM90-NEXT: @%p1 bra $L__BB369_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: st.param.b16 [func_retval0], %r17;
@@ -16842,14 +16531,14 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB378_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB370_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r2;
; SM90-NEXT: xor.b32 %r4, %r3, -1;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB378_1;
+; SM90-NEXT: @%p1 bra $L__BB370_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
@@ -16871,7 +16560,7 @@ define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB379_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB371_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
@@ -16889,7 +16578,7 @@ define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB379_1;
+; SM90-NEXT: @%p1 bra $L__BB371_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
@@ -16897,163 +16586,9 @@ define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
ret <8 x i8> %retval
}
-define <1 x i8> @nand_seq_cst_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v1i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<17>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v1i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.b8 %r6, [nand_seq_cst_v1i8_global_cta_param_1];
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: and.b32 %r8, %r7, 3;
-; SM90-NEXT: shl.b32 %r1, %r8, 3;
-; SM90-NEXT: mov.b32 %r9, 255;
-; SM90-NEXT: shl.b32 %r2, %r9, %r1;
-; SM90-NEXT: not.b32 %r3, %r2;
-; SM90-NEXT: shl.b32 %r4, %r6, %r1;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
-; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r10, %r16, %r4;
-; SM90-NEXT: not.b32 %r11, %r10;
-; SM90-NEXT: and.b32 %r12, %r11, %r2;
-; SM90-NEXT: and.b32 %r13, %r16, %r3;
-; SM90-NEXT: or.b32 %r14, %r13, %r12;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
-; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
-; SM90-NEXT: mov.b32 %r16, %r5;
-; SM90-NEXT: @%p1 bra $L__BB380_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r15, %r5, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r15;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") seq_cst
- ret <1 x i8> %retval
-}
-
-define <2 x i8> @nand_seq_cst_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v2i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_seq_cst_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_v2i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
-; SM90-NEXT: $L__BB381_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
-; SM90-NEXT: @%p1 bra $L__BB381_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") seq_cst
- ret <2 x i8> %retval
-}
-
-define <4 x i8> @nand_seq_cst_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v4i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<6>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_seq_cst_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB382_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r2;
-; SM90-NEXT: xor.b32 %r4, %r3, -1;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5;
-; SM90-NEXT: mov.b32 %r5, %r1;
-; SM90-NEXT: @%p1 bra $L__BB382_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") seq_cst
- ret <4 x i8> %retval
-}
-define <8 x i8> @nand_seq_cst_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
-; SM90-LABEL: nand_seq_cst_v8i8_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<9>;
-; SM90-NEXT: .reg .b64 %rd<12>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i8_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
-; SM90-NEXT: $L__BB383_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r7;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r8;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r8, %r2;
-; SM90-NEXT: and.b32 %r4, %r7, %r1;
-; SM90-NEXT: xor.b32 %r5, %r4, -1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r5;
-; SM90-NEXT: xor.b32 %r6, %r3, -1;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r6;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
-; SM90-NEXT: setp.ne.b64 %p1, %rd11, %rd6;
-; SM90-NEXT: mov.b64 {_, %r8}, %rd11;
-; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
-; SM90-NEXT: @%p1 bra $L__BB383_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") seq_cst
- ret <8 x i8> %retval
-}
+
+
define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM90-LABEL: nand_monotonic_v1i32_global_cta(
@@ -17066,14 +16601,14 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
; SM90-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB384_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r1;
; SM90-NEXT: not.b32 %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM90-NEXT: mov.b32 %r5, %r2;
-; SM90-NEXT: @%p1 bra $L__BB384_1;
+; SM90-NEXT: @%p1 bra $L__BB372_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
@@ -17094,7 +16629,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB385_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB373_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -17111,7 +16646,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB385_1;
+; SM90-NEXT: @%p1 bra $L__BB373_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
@@ -17141,7 +16676,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB386_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB374_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -17180,7 +16715,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB386_1;
+; SM90-NEXT: @%p1 bra $L__BB374_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
@@ -17211,7 +16746,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB387_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB375_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -17250,7 +16785,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB387_1;
+; SM90-NEXT: @%p1 bra $L__BB375_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -17264,7 +16799,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB387_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB375_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -17303,7 +16838,7 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB387_3;
+; SM90-NEXT: @%p2 bra $L__BB375_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -17323,14 +16858,14 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
; SM90-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB388_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r1;
; SM90-NEXT: not.b32 %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM90-NEXT: mov.b32 %r5, %r2;
-; SM90-NEXT: @%p1 bra $L__BB388_1;
+; SM90-NEXT: @%p1 bra $L__BB376_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.b32 [func_retval0], %r2;
@@ -17352,7 +16887,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB389_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB377_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -17369,7 +16904,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB389_1;
+; SM90-NEXT: @%p1 bra $L__BB377_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
@@ -17400,7 +16935,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB390_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB378_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -17439,7 +16974,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB390_1;
+; SM90-NEXT: @%p1 bra $L__BB378_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
@@ -17471,7 +17006,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB391_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB379_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -17510,7 +17045,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB391_1;
+; SM90-NEXT: @%p1 bra $L__BB379_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -17524,7 +17059,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB391_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB379_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -17563,7 +17098,7 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB391_3;
+; SM90-NEXT: @%p2 bra $L__BB379_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
@@ -17585,14 +17120,14 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB392_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: and.b32 %r3, %r5, %r1;
; SM90-NEXT: not.b32 %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
; SM90-NEXT: mov.b32 %r5, %r2;
-; SM90-NEXT: @%p1 bra $L__BB392_1;
+; SM90-NEXT: @%p1 bra $L__BB380_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
@@ -17614,7 +17149,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB393_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB381_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
@@ -17631,7 +17166,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB393_1;
+; SM90-NEXT: @%p1 bra $L__BB381_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
@@ -17662,7 +17197,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB394_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB382_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
@@ -17701,7 +17236,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB394_1;
+; SM90-NEXT: @%p1 bra $L__BB382_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
@@ -17733,7 +17268,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB395_1: // %atomicrmw.start
+; SM90-NEXT: $L__BB383_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
@@ -17772,7 +17307,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB395_1;
+; SM90-NEXT: @%p1 bra $L__BB383_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: mov.b64 %rd28, 0;
; SM90-NEXT: {
@@ -17786,7 +17321,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB395_3: // %atomicrmw.start2
+; SM90-NEXT: $L__BB383_3: // %atomicrmw.start2
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
@@ -17825,7 +17360,7 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB395_3;
+; SM90-NEXT: @%p2 bra $L__BB383_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
@@ -17833,268 +17368,3 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
}
-
-define <1 x i32> @nand_seq_cst_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v1i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<6>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v1i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.b32 %r1, [nand_seq_cst_v1i32_global_cta_param_1];
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
-; SM90-NEXT: $L__BB396_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: and.b32 %r3, %r5, %r1;
-; SM90-NEXT: not.b32 %r4, %r3;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
-; SM90-NEXT: setp.ne.b32 %p1, %r2, %r5;
-; SM90-NEXT: mov.b32 %r5, %r2;
-; SM90-NEXT: @%p1 bra $L__BB396_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") seq_cst
- ret <1 x i32> %retval
-}
-
-define <2 x i32> @nand_seq_cst_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v2i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<7>;
-; SM90-NEXT: .reg .b64 %rd<13>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_seq_cst_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v2i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
-; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: $L__BB397_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd3, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd4, %r6;
-; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
-; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
-; SM90-NEXT: and.b32 %r3, %r6, %r2;
-; SM90-NEXT: and.b32 %r4, %r5, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd7, %r4;
-; SM90-NEXT: cvt.u64.u32 %rd8, %r3;
-; SM90-NEXT: shl.b64 %rd9, %rd8, 32;
-; SM90-NEXT: or.b64 %rd10, %rd7, %rd9;
-; SM90-NEXT: not.b64 %rd11, %rd10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
-; SM90-NEXT: setp.ne.b64 %p1, %rd12, %rd6;
-; SM90-NEXT: mov.b64 {_, %r6}, %rd12;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
-; SM90-NEXT: @%p1 bra $L__BB397_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") seq_cst
- ret <2 x i32> %retval
-}
-
-define <4 x i32> @nand_seq_cst_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v4i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<13>;
-; SM90-NEXT: .reg .b64 %rd<28>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v4i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r12}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd4;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd3;
-; SM90-NEXT: $L__BB398_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd5, %r9;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r10;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r11;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r12;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: and.b32 %r5, %r12, %r4;
-; SM90-NEXT: and.b32 %r6, %r11, %r3;
-; SM90-NEXT: and.b32 %r7, %r10, %r2;
-; SM90-NEXT: and.b32 %r8, %r9, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r8;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r7;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: not.b64 %rd17, %rd16;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r6;
-; SM90-NEXT: cvt.u64.u32 %rd19, %r5;
-; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM90-NEXT: not.b64 %rd22, %rd21;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
-; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
-; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
-; SM90-NEXT: mov.b64 {_, %r12}, %rd24;
-; SM90-NEXT: cvt.u32.u64 %r11, %rd24;
-; SM90-NEXT: mov.b64 {_, %r10}, %rd23;
-; SM90-NEXT: cvt.u32.u64 %r9, %rd23;
-; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB398_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") seq_cst
- ret <4 x i32> %retval
-}
-
-define <8 x i32> @nand_seq_cst_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
-; SM90-LABEL: nand_seq_cst_v8i32_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<3>;
-; SM90-NEXT: .reg .b32 %r<25>;
-; SM90-NEXT: .reg .b64 %rd<54>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_v8i32_global_cta_param_0];
-; SM90-NEXT: fence.sc.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_seq_cst_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_seq_cst_v8i32_global_cta_param_1+16];
-; SM90-NEXT: mov.b64 %rd2, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd2, %rd2};
-; SM90-NEXT: mov.b128 swap, {%rd2, %rd2};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd3, %rd4}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r20}, %rd4;
-; SM90-NEXT: cvt.u32.u64 %r19, %rd4;
-; SM90-NEXT: mov.b64 {_, %r18}, %rd3;
-; SM90-NEXT: cvt.u32.u64 %r17, %rd3;
-; SM90-NEXT: $L__BB399_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd5, %r17;
-; SM90-NEXT: cvt.u64.u32 %rd6, %r18;
-; SM90-NEXT: shl.b64 %rd7, %rd6, 32;
-; SM90-NEXT: or.b64 %rd8, %rd5, %rd7;
-; SM90-NEXT: cvt.u64.u32 %rd9, %r19;
-; SM90-NEXT: cvt.u64.u32 %rd10, %r20;
-; SM90-NEXT: shl.b64 %rd11, %rd10, 32;
-; SM90-NEXT: or.b64 %rd12, %rd9, %rd11;
-; SM90-NEXT: and.b32 %r9, %r20, %r4;
-; SM90-NEXT: and.b32 %r10, %r19, %r3;
-; SM90-NEXT: and.b32 %r11, %r18, %r2;
-; SM90-NEXT: and.b32 %r12, %r17, %r1;
-; SM90-NEXT: cvt.u64.u32 %rd13, %r12;
-; SM90-NEXT: cvt.u64.u32 %rd14, %r11;
-; SM90-NEXT: shl.b64 %rd15, %rd14, 32;
-; SM90-NEXT: or.b64 %rd16, %rd13, %rd15;
-; SM90-NEXT: not.b64 %rd17, %rd16;
-; SM90-NEXT: cvt.u64.u32 %rd18, %r10;
-; SM90-NEXT: cvt.u64.u32 %rd19, %r9;
-; SM90-NEXT: shl.b64 %rd20, %rd19, 32;
-; SM90-NEXT: or.b64 %rd21, %rd18, %rd20;
-; SM90-NEXT: not.b64 %rd22, %rd21;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd8, %rd12};
-; SM90-NEXT: mov.b128 swap, {%rd17, %rd22};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd23, %rd24}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd25, %rd24, %rd12;
-; SM90-NEXT: xor.b64 %rd26, %rd23, %rd8;
-; SM90-NEXT: or.b64 %rd27, %rd26, %rd25;
-; SM90-NEXT: mov.b64 {_, %r20}, %rd24;
-; SM90-NEXT: cvt.u32.u64 %r19, %rd24;
-; SM90-NEXT: mov.b64 {_, %r18}, %rd23;
-; SM90-NEXT: cvt.u32.u64 %r17, %rd23;
-; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
-; SM90-NEXT: @%p1 bra $L__BB399_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: mov.b64 %rd28, 0;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd28, %rd28};
-; SM90-NEXT: mov.b128 swap, {%rd28, %rd28};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd29, %rd30}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: mov.b64 {_, %r24}, %rd30;
-; SM90-NEXT: cvt.u32.u64 %r23, %rd30;
-; SM90-NEXT: mov.b64 {_, %r22}, %rd29;
-; SM90-NEXT: cvt.u32.u64 %r21, %rd29;
-; SM90-NEXT: $L__BB399_3: // %atomicrmw.start2
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: cvt.u64.u32 %rd31, %r21;
-; SM90-NEXT: cvt.u64.u32 %rd32, %r22;
-; SM90-NEXT: shl.b64 %rd33, %rd32, 32;
-; SM90-NEXT: or.b64 %rd34, %rd31, %rd33;
-; SM90-NEXT: cvt.u64.u32 %rd35, %r23;
-; SM90-NEXT: cvt.u64.u32 %rd36, %r24;
-; SM90-NEXT: shl.b64 %rd37, %rd36, 32;
-; SM90-NEXT: or.b64 %rd38, %rd35, %rd37;
-; SM90-NEXT: and.b32 %r13, %r24, %r8;
-; SM90-NEXT: and.b32 %r14, %r23, %r7;
-; SM90-NEXT: and.b32 %r15, %r22, %r6;
-; SM90-NEXT: and.b32 %r16, %r21, %r5;
-; SM90-NEXT: cvt.u64.u32 %rd39, %r16;
-; SM90-NEXT: cvt.u64.u32 %rd40, %r15;
-; SM90-NEXT: shl.b64 %rd41, %rd40, 32;
-; SM90-NEXT: or.b64 %rd42, %rd39, %rd41;
-; SM90-NEXT: not.b64 %rd43, %rd42;
-; SM90-NEXT: cvt.u64.u32 %rd44, %r14;
-; SM90-NEXT: cvt.u64.u32 %rd45, %r13;
-; SM90-NEXT: shl.b64 %rd46, %rd45, 32;
-; SM90-NEXT: or.b64 %rd47, %rd44, %rd46;
-; SM90-NEXT: not.b64 %rd48, %rd47;
-; SM90-NEXT: {
-; SM90-NEXT: .reg .b128 cmp, swap, dst;
-; SM90-NEXT: mov.b128 cmp, {%rd34, %rd38};
-; SM90-NEXT: mov.b128 swap, {%rd43, %rd48};
-; SM90-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1+16], cmp, swap;
-; SM90-NEXT: mov.b128 {%rd49, %rd50}, dst;
-; SM90-NEXT: }
-; SM90-NEXT: xor.b64 %rd51, %rd50, %rd38;
-; SM90-NEXT: xor.b64 %rd52, %rd49, %rd34;
-; SM90-NEXT: or.b64 %rd53, %rd52, %rd51;
-; SM90-NEXT: mov.b64 {_, %r24}, %rd50;
-; SM90-NEXT: cvt.u32.u64 %r23, %rd50;
-; SM90-NEXT: mov.b64 {_, %r22}, %rd49;
-; SM90-NEXT: cvt.u32.u64 %r21, %rd49;
-; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
-; SM90-NEXT: @%p2 bra $L__BB399_3;
-; SM90-NEXT: // %bb.4: // %atomicrmw.end1
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
-; SM90-NEXT: ret;
- %retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") seq_cst
- ret <8 x i32> %retval
-}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index 14825faa144c4..cb8b7a7a54af1 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -104,26 +104,6 @@ entry:
ret <4 x i32> %old
}
-; Seq-cst elementwise expansion also gets one outer fence before splitting.
-define <2 x i32> @add_seq_cst_v4i32_elementwise(ptr %addr, <2 x i32> %val) {
-; CHECK-LABEL: @add_seq_cst_v4i32_elementwise(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: fence syncscope("block") seq_cst
-; CHECK-NEXT: [[LO_VAL2:%.*]] = extractelement <2 x i32> [[LO_VAL:%.*]], i64 0
-; CHECK-NEXT: [[HI_VAL3:%.*]] = extractelement <2 x i32> [[LO_VAL]], i64 1
-; CHECK-NEXT: [[HI_PTR4:%.*]] = getelementptr inbounds i32, ptr [[ADDR:%.*]], i64 1
-; CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[ADDR]], i32 [[LO_VAL2]] syncscope("block") monotonic, align 16
-; CHECK-NEXT: [[TMP3:%.*]] = atomicrmw add ptr [[HI_PTR4]], i32 [[HI_VAL3]] syncscope("block") monotonic, align 4
-; CHECK-NEXT: [[LO_OLD8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[HI_OLD9:%.*]] = insertelement <2 x i32> [[LO_OLD8]], i32 [[TMP3]], i64 1
-; CHECK-NEXT: fence syncscope("block") acquire
-; CHECK-NEXT: ret <2 x i32> [[HI_OLD9]]
-;
-entry:
- %old = atomicrmw elementwise add ptr %addr, <2 x i32> %val syncscope("block") seq_cst, align 16
- ret <2 x i32> %old
-}
-
; Whole-vector cmpxchg expansion: sm_90 + ptx83 supports 128-bit cmpxchg, so
; this vector can stay whole during AtomicExpand.
define <8 x half> @fmin_v8f16_elementwise(ptr %addr, <8 x half> %val) {
>From 5a3efa62bcdf5a686f49f9452796cd6b91412581 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 4 Sep 2026 01:08:41 +0000
Subject: [PATCH 13/15] mips comment
---
.../AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
index b2b30dfe24d06..f38c5dbdab1a1 100644
--- a/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/Mips/expand-atomic-rmw-elementwise.ll
@@ -1,7 +1,7 @@
; RUN: opt -S -mtriple=mips64-mti-linux-gnu -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s
; MIPS uses TargetLowering's default AtomicRMW expansion, so an elementwise
-; atomicrmw is expanded into a compare-exchange loop.
+; atomicrmw is scalarized.
define <2 x i32> @atomicrmw_add_elementwise(ptr %ptr, <2 x i32> %value) {
; CHECK-LABEL: define <2 x i32> @atomicrmw_add_elementwise(
; CHECK-SAME: ptr [[PTR:%.*]], <2 x i32> [[VALUE:%.*]]) {
>From 5ccd438c5d40e83f0564a6ec90656a61655bfe0d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 4 Sep 2026 01:44:53 +0000
Subject: [PATCH 14/15] update tests
---
.../NVPTX/atomicrmw-elementwise-sm60.ll | 512 ++-
.../NVPTX/atomicrmw-elementwise-sm70.ll | 512 ++-
.../NVPTX/atomicrmw-elementwise-sm90.ll | 3124 ++++++++---------
3 files changed, 2020 insertions(+), 2128 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
index bf5d392ff0b9b..241b17a491bb6 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm60.ll
@@ -531,31 +531,29 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB19_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -563,11 +561,13 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -1133,31 +1133,29 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB35_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: sub.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: sub.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: sub.s16 %rs12, %rs11, %rs10;
@@ -1165,11 +1163,13 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
; SM60-NEXT: sub.s16 %rs15, %rs14, %rs13;
; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -2100,46 +2100,37 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-LABEL: nand_acq_rel_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM60-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r2, %r8, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: ld.param.b16 %r9, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r9, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
; SM60-NEXT: @%p1 bra $L__BB65_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -3832,16 +3823,16 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM60-NEXT: $L__BB114_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM60-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM60-NEXT: setp.gt.s32 %p4, %r10, %r9;
; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -3886,16 +3877,16 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: $L__BB115_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM60-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM60-NEXT: setp.gt.s32 %p2, %r6, %r5;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4467,16 +4458,16 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM60-NEXT: $L__BB130_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
; SM60-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM60-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM60-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM60-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM60-NEXT: setp.le.s32 %p4, %r10, %r9;
; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -4521,16 +4512,16 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: $L__BB131_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r49;
; SM60-NEXT: cvt.u64.u32 %rd4, %r50;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM60-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM60-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM60-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM60-NEXT: setp.le.s32 %p2, %r6, %r5;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -5101,16 +5092,16 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM60-NEXT: $L__BB146_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM60-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM60-NEXT: setp.gt.u32 %p4, %r10, %r9;
; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5147,18 +5138,16 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: $L__BB147_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM60-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM60-NEXT: setp.gt.u32 %p2, %r6, %r5;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5179,7 +5168,9 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM60-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -5711,16 +5702,16 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM60-NEXT: $L__BB162_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
; SM60-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM60-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM60-NEXT: setp.le.u32 %p4, %r10, %r9;
; SM60-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5757,18 +5748,16 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: $L__BB163_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM60-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM60-NEXT: setp.le.u32 %p2, %r6, %r5;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5789,7 +5778,9 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM60-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM60-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -6328,8 +6319,6 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM60-NEXT: $L__BB178_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6346,7 +6335,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: add.s16 %rs8, %rs7, 1;
; SM60-NEXT: setp.ge.u32 %p1, %r3, %r7;
; SM60-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM60-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM60-NEXT: setp.ge.u32 %p4, %r6, %r10;
; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
; SM60-NEXT: selp.b32 %r12, 0, %r11, %p4;
@@ -6386,8 +6377,6 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM60-NEXT: $L__BB179_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -6418,7 +6407,9 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs15, %r10;
; SM60-NEXT: add.s16 %rs16, %rs15, 1;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM60-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM60-NEXT: setp.ge.u32 %p2, %r4, %r12;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM60-NEXT: setp.ge.u32 %p3, %r5, %r13;
@@ -7147,8 +7138,6 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.volatile.global.b32 %r26, [%rd1];
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM60-NEXT: $L__BB194_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7169,7 +7158,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.eq.b32 %p4, %r6, 0;
; SM60-NEXT: setp.gt.u32 %p5, %r3, %r7;
; SM60-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM60-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM60-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM60-NEXT: setp.gt.u32 %p8, %r6, %r10;
; SM60-NEXT: cvt.u32.u16 %r11, %rs8;
; SM60-NEXT: selp.b32 %r12, %r10, %r11, %p8;
@@ -7213,10 +7204,6 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs17, %r11;
-; SM60-NEXT: cvt.u16.u32 %rs18, %r12;
; SM60-NEXT: $L__BB195_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -7255,7 +7242,9 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.eq.b32 %p6, %r8, 0;
; SM60-NEXT: setp.eq.b32 %p7, %r9, 0;
; SM60-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM60-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM60-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM60-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM60-NEXT: setp.gt.u32 %p10, %r4, %r12;
; SM60-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM60-NEXT: setp.gt.u32 %p11, %r5, %r13;
@@ -7269,6 +7258,8 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: setp.gt.u32 %p15, %r9, %r17;
; SM60-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
; SM60-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM60-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM60-NEXT: cvt.u16.u32 %rs18, %r12;
; SM60-NEXT: cvt.u16.u32 %rs19, %r13;
; SM60-NEXT: cvt.u16.u32 %rs20, %r14;
; SM60-NEXT: cvt.u16.u32 %rs21, %r15;
@@ -8052,10 +8043,6 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: ld.volatile.global.b32 %r22, [%rd1];
; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r9;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r7;
; SM60-NEXT: cvt.u16.u32 %rs3, %r5;
; SM60-NEXT: cvt.u16.u32 %rs4, %r3;
; SM60-NEXT: $L__BB210_1: // %atomicrmw.start
@@ -8064,10 +8051,14 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
; SM60-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM60-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
; SM60-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM60-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM60-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
; SM60-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM60-NEXT: cvt.u16.u32 %rs2, %r7;
; SM60-NEXT: cvt.u16.u32 %rs5, %r4;
; SM60-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u16.u32 %rs7, %r6;
@@ -8114,18 +8105,16 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs13, %r5;
-; SM60-NEXT: cvt.u16.u32 %rs15, %r3;
; SM60-NEXT: $L__BB211_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: cvt.u64.u32 %rd3, %r33;
; SM60-NEXT: cvt.u64.u32 %rd4, %r34;
; SM60-NEXT: shl.b64 %rd5, %rd4, 32;
; SM60-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM60-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM60-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM60-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM60-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM60-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM60-NEXT: setp.ge.u32 %p2, %r6, %r5;
; SM60-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8158,7 +8147,9 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: cvt.u16.u32 %rs10, %r10;
; SM60-NEXT: cvt.u16.u32 %rs11, %r7;
; SM60-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r5;
; SM60-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM60-NEXT: cvt.u16.u32 %rs15, %r3;
; SM60-NEXT: cvt.u16.u32 %rs16, %r4;
; SM60-NEXT: sub.s16 %rs17, %rs16, %rs15;
; SM60-NEXT: sub.s16 %rs18, %rs14, %rs13;
@@ -9114,34 +9105,32 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB227_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: max.u16 %rs3, %rs2, %rs1;
; SM60-NEXT: sub.s16 %rs4, %rs3, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs5, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs6, %r7;
; SM60-NEXT: max.u16 %rs7, %rs6, %rs5;
; SM60-NEXT: sub.s16 %rs8, %rs7, %rs5;
; SM60-NEXT: cvt.u32.u16 %r8, %rs8;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs9, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs10, %r11;
; SM60-NEXT: max.u16 %rs11, %rs10, %rs9;
; SM60-NEXT: sub.s16 %rs12, %rs11, %rs9;
; SM60-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs13, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r14;
; SM60-NEXT: max.u16 %rs15, %rs14, %rs13;
@@ -9150,12 +9139,14 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs18, %r19;
; SM60-NEXT: max.u16 %rs19, %rs18, %rs17;
; SM60-NEXT: sub.s16 %rs20, %rs19, %rs17;
; SM60-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs21, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs22, %r22;
@@ -13092,16 +13083,15 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
-; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
-; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
; SM60-NEXT: $L__BB298_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
; SM60-NEXT: min.f32 %r5, %r4, %r3;
; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
; SM60-NEXT: min.f32 %r8, %r7, %r6;
; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
@@ -13111,6 +13101,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
; SM60-NEXT: min.f32 %r12, %r11, %r10;
; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
; SM60-NEXT: min.f32 %r15, %r14, %r13;
; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
@@ -13151,11 +13142,10 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
; SM60-NEXT: $L__BB299_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
; SM60-NEXT: min.f32 %r7, %r6, %r5;
@@ -13166,6 +13156,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
; SM60-NEXT: min.f32 %r14, %r13, %r12;
@@ -13192,11 +13183,10 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
; SM60-NEXT: $L__BB299_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
; SM60-NEXT: min.f32 %r21, %r20, %r19;
@@ -13207,6 +13197,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
; SM60-NEXT: min.f32 %r28, %r27, %r26;
@@ -13336,16 +13327,15 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: cvt.u32.u64 %r17, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.f32.f16 %r3, %rs2;
-; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM60-NEXT: cvt.f32.f16 %r10, %rs8;
-; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
; SM60-NEXT: $L__BB302_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM60-NEXT: cvt.f32.f16 %r4, %rs4;
; SM60-NEXT: max.f32 %r5, %r4, %r3;
; SM60-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM60-NEXT: cvt.f32.f16 %r6, %rs1;
; SM60-NEXT: cvt.f32.f16 %r7, %rs3;
; SM60-NEXT: max.f32 %r8, %r7, %r6;
; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r8;
@@ -13355,6 +13345,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM60-NEXT: cvt.f32.f16 %r11, %rs10;
; SM60-NEXT: max.f32 %r12, %r11, %r10;
; SM60-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM60-NEXT: cvt.f32.f16 %r13, %rs7;
; SM60-NEXT: cvt.f32.f16 %r14, %rs9;
; SM60-NEXT: max.f32 %r15, %r14, %r13;
; SM60-NEXT: cvt.rn.f16.f32 %rs12, %r15;
@@ -13395,11 +13386,10 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
; SM60-NEXT: $L__BB303_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.f32.f16 %r5, %rs2;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM60-NEXT: cvt.f32.f16 %r6, %rs4;
; SM60-NEXT: max.f32 %r7, %r6, %r5;
@@ -13410,6 +13400,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.rn.f16.f32 %rs6, %r10;
; SM60-NEXT: mov.b32 %r11, {%rs6, %rs5};
; SM60-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM60-NEXT: cvt.f32.f16 %r12, %rs8;
; SM60-NEXT: mov.b32 {%rs9, %rs10}, %r34;
; SM60-NEXT: cvt.f32.f16 %r13, %rs10;
; SM60-NEXT: max.f32 %r14, %r13, %r12;
@@ -13436,11 +13427,10 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM60-NEXT: cvt.u32.u64 %r35, %rd12;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
; SM60-NEXT: $L__BB303_3: // %atomicrmw.start2
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.f32.f16 %r19, %rs14;
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM60-NEXT: cvt.f32.f16 %r20, %rs16;
; SM60-NEXT: max.f32 %r21, %r20, %r19;
@@ -13451,6 +13441,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM60-NEXT: cvt.rn.f16.f32 %rs18, %r24;
; SM60-NEXT: mov.b32 %r25, {%rs18, %rs17};
; SM60-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM60-NEXT: cvt.f32.f16 %r26, %rs20;
; SM60-NEXT: mov.b32 {%rs21, %rs22}, %r36;
; SM60-NEXT: cvt.f32.f16 %r27, %rs22;
; SM60-NEXT: max.f32 %r28, %r27, %r26;
@@ -14208,13 +14199,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: $L__BB314_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14227,6 +14214,8 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM60-NEXT: or.b32 %r11, %r7, 4194304;
; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
; SM60-NEXT: shl.b32 %r16, %r15, 16;
; SM60-NEXT: add.rn.f32 %r17, %r16, %r14;
@@ -14248,6 +14237,8 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM60-NEXT: or.b32 %r32, %r28, 4194304;
; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
; SM60-NEXT: shl.b32 %r37, %r36, 16;
; SM60-NEXT: add.rn.f32 %r38, %r37, %r35;
@@ -14294,13 +14285,11 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: $L__BB315_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
@@ -14324,6 +14313,8 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
; SM60-NEXT: shl.b32 %r29, %r28, 16;
@@ -14555,13 +14546,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: $L__BB318_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14574,6 +14561,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM60-NEXT: or.b32 %r11, %r7, 4194304;
; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
; SM60-NEXT: shl.b32 %r16, %r15, 16;
; SM60-NEXT: sub.rn.f32 %r17, %r16, %r14;
@@ -14595,6 +14584,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM60-NEXT: or.b32 %r32, %r28, 4194304;
; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
; SM60-NEXT: shl.b32 %r37, %r36, 16;
; SM60-NEXT: sub.rn.f32 %r38, %r37, %r35;
@@ -14641,13 +14632,11 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: $L__BB319_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
@@ -14671,6 +14660,8 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
; SM60-NEXT: shl.b32 %r29, %r28, 16;
@@ -14902,13 +14893,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: $L__BB322_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14921,6 +14908,8 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM60-NEXT: or.b32 %r11, %r7, 4194304;
; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
; SM60-NEXT: shl.b32 %r16, %r15, 16;
; SM60-NEXT: min.f32 %r17, %r16, %r14;
@@ -14942,6 +14931,8 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM60-NEXT: or.b32 %r32, %r28, 4194304;
; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
; SM60-NEXT: shl.b32 %r37, %r36, 16;
; SM60-NEXT: min.f32 %r38, %r37, %r35;
@@ -14988,13 +14979,11 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: $L__BB323_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
@@ -15018,6 +15007,8 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
; SM60-NEXT: shl.b32 %r29, %r28, 16;
@@ -15249,13 +15240,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM60-NEXT: cvt.u32.u16 %r24, %rs6;
; SM60-NEXT: shl.b32 %r25, %r24, 16;
-; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: $L__BB326_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -15268,6 +15255,8 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM60-NEXT: or.b32 %r11, %r7, 4194304;
; SM60-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM60-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM60-NEXT: shl.b32 %r14, %r13, 16;
; SM60-NEXT: cvt.u32.u16 %r15, %rs3;
; SM60-NEXT: shl.b32 %r16, %r15, 16;
; SM60-NEXT: max.f32 %r17, %r16, %r14;
@@ -15289,6 +15278,8 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM60-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM60-NEXT: or.b32 %r32, %r28, 4194304;
; SM60-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM60-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM60-NEXT: shl.b32 %r35, %r34, 16;
; SM60-NEXT: cvt.u32.u16 %r36, %rs7;
; SM60-NEXT: shl.b32 %r37, %r36, 16;
; SM60-NEXT: max.f32 %r38, %r37, %r35;
@@ -15335,13 +15326,11 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r89, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: $L__BB327_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
@@ -15365,6 +15354,8 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM60-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM60-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM60-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM60-NEXT: shl.b32 %r27, %r26, 16;
; SM60-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM60-NEXT: cvt.u32.u16 %r28, %rs8;
; SM60-NEXT: shl.b32 %r29, %r28, 16;
@@ -15600,13 +15591,9 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: cvt.u32.u16 %r12, %rs14;
; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
-; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: $L__BB330_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
@@ -15622,6 +15609,8 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
; SM60-NEXT: setp.eq.b16 %p5, %rs3, -32768;
; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
; SM60-NEXT: setp.lt.f32 %p6, %r10, %r8;
@@ -15645,6 +15634,8 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
; SM60-NEXT: setp.eq.b16 %p13, %rs15, -32768;
; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: cvt.u32.u16 %r18, %rs15;
; SM60-NEXT: shl.b32 %r19, %r18, 16;
; SM60-NEXT: setp.lt.f32 %p14, %r19, %r17;
@@ -15690,16 +15681,14 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
-; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: $L__BB331_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r41;
; SM60-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: setp.lt.f32 %p2, %r8, %r6;
@@ -15725,6 +15714,8 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r42;
; SM60-NEXT: setp.eq.b16 %p9, %rs16, -32768;
; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: setp.lt.f32 %p10, %r17, %r15;
@@ -15963,13 +15954,9 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM60-NEXT: cvt.u32.u16 %r3, %rs2;
; SM60-NEXT: shl.b32 %r4, %r3, 16;
-; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
-; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM60-NEXT: cvt.u32.u16 %r12, %rs14;
; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
-; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: $L__BB334_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r21;
@@ -15985,6 +15972,8 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
; SM60-NEXT: setp.eq.b16 %p5, %rs3, 0;
; SM60-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM60-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
; SM60-NEXT: shl.b32 %r10, %r9, 16;
; SM60-NEXT: setp.gt.f32 %p6, %r10, %r8;
@@ -16008,6 +15997,8 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
; SM60-NEXT: setp.eq.b16 %p13, %rs15, 0;
; SM60-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM60-NEXT: cvt.u32.u16 %r16, %rs13;
+; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: cvt.u32.u16 %r18, %rs15;
; SM60-NEXT: shl.b32 %r19, %r18, 16;
; SM60-NEXT: setp.gt.f32 %p14, %r19, %r17;
@@ -16053,16 +16044,14 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r41, %rd2;
; SM60-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: mov.b32 {%rs13, %rs14}, %r2;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
-; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: $L__BB335_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
; SM60-NEXT: mov.b32 {%rs3, %rs4}, %r41;
; SM60-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM60-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM60-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM60-NEXT: shl.b32 %r6, %r5, 16;
; SM60-NEXT: cvt.u32.u16 %r7, %rs4;
; SM60-NEXT: shl.b32 %r8, %r7, 16;
; SM60-NEXT: setp.gt.f32 %p2, %r8, %r6;
@@ -16088,6 +16077,8 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM60-NEXT: mov.b32 {%rs15, %rs16}, %r42;
; SM60-NEXT: setp.eq.b16 %p9, %rs16, 0;
; SM60-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM60-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM60-NEXT: shl.b32 %r15, %r14, 16;
; SM60-NEXT: cvt.u32.u16 %r16, %rs16;
; SM60-NEXT: shl.b32 %r17, %r16, 16;
; SM60-NEXT: setp.gt.f32 %p10, %r17, %r15;
@@ -16352,31 +16343,29 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB339_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16384,11 +16373,13 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -16581,31 +16572,29 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB343_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16613,11 +16602,13 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -16812,31 +16803,29 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
; SM60-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM60-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: $L__BB347_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM60-NEXT: cvt.u16.u32 %rs1, %r3;
; SM60-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs2, %r4;
; SM60-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM60-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM60-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM60-NEXT: cvt.u16.u32 %rs4, %r6;
; SM60-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs5, %r7;
; SM60-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM60-NEXT: cvt.u32.u16 %r8, %rs6;
; SM60-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM60-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM60-NEXT: cvt.u16.u32 %rs7, %r10;
; SM60-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM60-NEXT: cvt.u16.u32 %rs8, %r11;
; SM60-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM60-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM60-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM60-NEXT: cvt.u16.u32 %rs10, %r13;
; SM60-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM60-NEXT: cvt.u16.u32 %rs11, %r14;
; SM60-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16844,11 +16833,13 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM60-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM60-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM60-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM60-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs13, %r18;
; SM60-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM60-NEXT: cvt.u16.u32 %rs14, %r19;
; SM60-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM60-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM60-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs16, %r21;
; SM60-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM60-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -17177,44 +17168,35 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM60-LABEL: nand_monotonic_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM60-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r2, %r8, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: ld.param.b16 %r9, [nand_monotonic_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r9, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
; SM60-NEXT: $L__BB361_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
; SM60-NEXT: @%p1 bra $L__BB361_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
@@ -17328,45 +17310,36 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-LABEL: nand_acquire_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM60-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r2, %r8, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: ld.param.b16 %r9, [nand_acquire_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r9, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
; SM60-NEXT: $L__BB365_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
; SM60-NEXT: @%p1 bra $L__BB365_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
ret <2 x i8> %retval
@@ -17482,45 +17455,36 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM60-LABEL: nand_release_v2i8_global_cta(
; SM60: {
; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<19>;
+; SM60-NEXT: .reg .b32 %r<17>;
; SM60-NEXT: .reg .b64 %rd<3>;
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM60-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM60-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM60-NEXT: membar.cta;
; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM60-NEXT: and.b32 %r6, %r5, 3;
-; SM60-NEXT: shl.b32 %r1, %r6, 3;
-; SM60-NEXT: mov.b32 %r7, 65535;
-; SM60-NEXT: shl.b32 %r8, %r7, %r1;
-; SM60-NEXT: not.b32 %r2, %r8;
-; SM60-NEXT: ld.volatile.global.b32 %r18, [%rd1];
+; SM60-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM60-NEXT: and.b32 %r7, %r6, 3;
+; SM60-NEXT: shl.b32 %r1, %r7, 3;
+; SM60-NEXT: mov.b32 %r8, 65535;
+; SM60-NEXT: shl.b32 %r2, %r8, %r1;
+; SM60-NEXT: not.b32 %r3, %r2;
+; SM60-NEXT: ld.param.b16 %r9, [nand_release_v2i8_global_cta_param_1];
+; SM60-NEXT: shl.b32 %r4, %r9, %r1;
+; SM60-NEXT: ld.volatile.global.b32 %r16, [%rd1];
; SM60-NEXT: $L__BB369_1: // %atomicrmw.start
; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r9, %r18, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM60-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM60-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM60-NEXT: and.b32 %r11, %r10, %r4;
-; SM60-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM60-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM60-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM60-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM60-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM60-NEXT: shl.b32 %r14, %r13, %r1;
-; SM60-NEXT: and.b32 %r15, %r18, %r2;
-; SM60-NEXT: or.b32 %r16, %r15, %r14;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM60-NEXT: mov.b32 %r18, %r3;
+; SM60-NEXT: and.b32 %r10, %r16, %r4;
+; SM60-NEXT: not.b32 %r11, %r10;
+; SM60-NEXT: and.b32 %r12, %r11, %r2;
+; SM60-NEXT: and.b32 %r13, %r16, %r3;
+; SM60-NEXT: or.b32 %r14, %r13, %r12;
+; SM60-NEXT: atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM60-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM60-NEXT: mov.b32 %r16, %r5;
; SM60-NEXT: @%p1 bra $L__BB369_1;
; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r17, %r3, %r1;
-; SM60-NEXT: st.param.b16 [func_retval0], %r17;
+; SM60-NEXT: shr.u32 %r15, %r5, %r1;
+; SM60-NEXT: st.param.b16 [func_retval0], %r15;
; SM60-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
index 32152afa7c416..c07929cf32611 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll
@@ -531,31 +531,29 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB19_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -563,11 +561,13 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -1133,31 +1133,29 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB35_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: sub.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: sub.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: sub.s16 %rs12, %rs11, %rs10;
@@ -1165,11 +1163,13 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
; SM70-NEXT: sub.s16 %rs15, %rs14, %rs13;
; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -2100,46 +2100,37 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-LABEL: nand_acq_rel_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r2, %r8, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: ld.param.b16 %r9, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r9, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
; SM70-NEXT: @%p1 bra $L__BB65_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -3832,16 +3823,16 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT: $L__BB114_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM70-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM70-NEXT: setp.gt.s32 %p4, %r10, %r9;
; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -3886,16 +3877,16 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: $L__BB115_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM70-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM70-NEXT: setp.gt.s32 %p2, %r6, %r5;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4467,16 +4458,16 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT: $L__BB130_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
; SM70-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM70-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM70-NEXT: setp.le.s32 %p4, %r10, %r9;
; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -4521,16 +4512,16 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: $L__BB131_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r49;
; SM70-NEXT: cvt.u64.u32 %rd4, %r50;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM70-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM70-NEXT: setp.le.s32 %p2, %r6, %r5;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -5101,16 +5092,16 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT: $L__BB146_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM70-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM70-NEXT: setp.gt.u32 %p4, %r10, %r9;
; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5147,18 +5138,16 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: $L__BB147_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT: setp.gt.u32 %p2, %r6, %r5;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5179,7 +5168,9 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM70-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -5711,16 +5702,16 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT: $L__BB162_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
; SM70-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM70-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM70-NEXT: setp.le.u32 %p4, %r10, %r9;
; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5757,18 +5748,16 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: $L__BB163_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT: setp.le.u32 %p2, %r6, %r5;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5789,7 +5778,9 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM70-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM70-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -6328,8 +6319,6 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT: $L__BB178_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6346,7 +6335,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: add.s16 %rs8, %rs7, 1;
; SM70-NEXT: setp.ge.u32 %p1, %r3, %r7;
; SM70-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT: setp.ge.u32 %p4, %r6, %r10;
; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
; SM70-NEXT: selp.b32 %r12, 0, %r11, %p4;
@@ -6386,8 +6377,6 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT: $L__BB179_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -6418,7 +6407,9 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs15, %r10;
; SM70-NEXT: add.s16 %rs16, %rs15, 1;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT: setp.ge.u32 %p2, %r4, %r12;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT: setp.ge.u32 %p3, %r5, %r13;
@@ -7147,8 +7138,6 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT: $L__BB194_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7169,7 +7158,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.eq.b32 %p4, %r6, 0;
; SM70-NEXT: setp.gt.u32 %p5, %r3, %r7;
; SM70-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM70-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT: setp.gt.u32 %p8, %r6, %r10;
; SM70-NEXT: cvt.u32.u16 %r11, %rs8;
; SM70-NEXT: selp.b32 %r12, %r10, %r11, %p8;
@@ -7213,10 +7204,6 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs17, %r11;
-; SM70-NEXT: cvt.u16.u32 %rs18, %r12;
; SM70-NEXT: $L__BB195_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -7255,7 +7242,9 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.eq.b32 %p6, %r8, 0;
; SM70-NEXT: setp.eq.b32 %p7, %r9, 0;
; SM70-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM70-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM70-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT: setp.gt.u32 %p10, %r4, %r12;
; SM70-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT: setp.gt.u32 %p11, %r5, %r13;
@@ -7269,6 +7258,8 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: setp.gt.u32 %p15, %r9, %r17;
; SM70-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
; SM70-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM70-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM70-NEXT: cvt.u16.u32 %rs18, %r12;
; SM70-NEXT: cvt.u16.u32 %rs19, %r13;
; SM70-NEXT: cvt.u16.u32 %rs20, %r14;
; SM70-NEXT: cvt.u16.u32 %rs21, %r15;
@@ -8052,10 +8043,6 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r9;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r7;
; SM70-NEXT: cvt.u16.u32 %rs3, %r5;
; SM70-NEXT: cvt.u16.u32 %rs4, %r3;
; SM70-NEXT: $L__BB210_1: // %atomicrmw.start
@@ -8064,10 +8051,14 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
; SM70-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
; SM70-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM70-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
; SM70-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM70-NEXT: cvt.u16.u32 %rs2, %r7;
; SM70-NEXT: cvt.u16.u32 %rs5, %r4;
; SM70-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u16.u32 %rs7, %r6;
@@ -8114,18 +8105,16 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs13, %r5;
-; SM70-NEXT: cvt.u16.u32 %rs15, %r3;
; SM70-NEXT: $L__BB211_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: cvt.u64.u32 %rd3, %r33;
; SM70-NEXT: cvt.u64.u32 %rd4, %r34;
; SM70-NEXT: shl.b64 %rd5, %rd4, 32;
; SM70-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM70-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM70-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT: setp.ge.u32 %p2, %r6, %r5;
; SM70-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8158,7 +8147,9 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: cvt.u16.u32 %rs10, %r10;
; SM70-NEXT: cvt.u16.u32 %rs11, %r7;
; SM70-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r5;
; SM70-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM70-NEXT: cvt.u16.u32 %rs15, %r3;
; SM70-NEXT: cvt.u16.u32 %rs16, %r4;
; SM70-NEXT: sub.s16 %rs17, %rs16, %rs15;
; SM70-NEXT: sub.s16 %rs18, %rs14, %rs13;
@@ -9114,34 +9105,32 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB227_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: max.u16 %rs3, %rs2, %rs1;
; SM70-NEXT: sub.s16 %rs4, %rs3, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs5, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs6, %r7;
; SM70-NEXT: max.u16 %rs7, %rs6, %rs5;
; SM70-NEXT: sub.s16 %rs8, %rs7, %rs5;
; SM70-NEXT: cvt.u32.u16 %r8, %rs8;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs9, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs10, %r11;
; SM70-NEXT: max.u16 %rs11, %rs10, %rs9;
; SM70-NEXT: sub.s16 %rs12, %rs11, %rs9;
; SM70-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs13, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r14;
; SM70-NEXT: max.u16 %rs15, %rs14, %rs13;
@@ -9150,12 +9139,14 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs18, %r19;
; SM70-NEXT: max.u16 %rs19, %rs18, %rs17;
; SM70-NEXT: sub.s16 %rs20, %rs19, %rs17;
; SM70-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs21, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs22, %r22;
@@ -13016,16 +13007,15 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
-; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
-; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
; SM70-NEXT: $L__BB298_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
; SM70-NEXT: min.f32 %r5, %r4, %r3;
; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
; SM70-NEXT: min.f32 %r8, %r7, %r6;
; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
@@ -13035,6 +13025,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
; SM70-NEXT: min.f32 %r12, %r11, %r10;
; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
; SM70-NEXT: min.f32 %r15, %r14, %r13;
; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
@@ -13075,11 +13066,10 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
; SM70-NEXT: $L__BB299_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
; SM70-NEXT: min.f32 %r7, %r6, %r5;
@@ -13090,6 +13080,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
; SM70-NEXT: min.f32 %r14, %r13, %r12;
@@ -13116,11 +13107,10 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
; SM70-NEXT: $L__BB299_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
; SM70-NEXT: min.f32 %r21, %r20, %r19;
@@ -13131,6 +13121,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
; SM70-NEXT: min.f32 %r28, %r27, %r26;
@@ -13260,16 +13251,15 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: cvt.u32.u64 %r17, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.f32.f16 %r3, %rs2;
-; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT: cvt.f32.f16 %r10, %rs8;
-; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
; SM70-NEXT: $L__BB302_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r17;
; SM70-NEXT: cvt.f32.f16 %r4, %rs4;
; SM70-NEXT: max.f32 %r5, %r4, %r3;
; SM70-NEXT: cvt.rn.f16.f32 %rs5, %r5;
+; SM70-NEXT: cvt.f32.f16 %r6, %rs1;
; SM70-NEXT: cvt.f32.f16 %r7, %rs3;
; SM70-NEXT: max.f32 %r8, %r7, %r6;
; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r8;
@@ -13279,6 +13269,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM70-NEXT: cvt.f32.f16 %r11, %rs10;
; SM70-NEXT: max.f32 %r12, %r11, %r10;
; SM70-NEXT: cvt.rn.f16.f32 %rs11, %r12;
+; SM70-NEXT: cvt.f32.f16 %r13, %rs7;
; SM70-NEXT: cvt.f32.f16 %r14, %rs9;
; SM70-NEXT: max.f32 %r15, %r14, %r13;
; SM70-NEXT: cvt.rn.f16.f32 %rs12, %r15;
@@ -13319,11 +13310,10 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r2;
-; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
; SM70-NEXT: $L__BB303_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.f32.f16 %r5, %rs2;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r33;
; SM70-NEXT: cvt.f32.f16 %r6, %rs4;
; SM70-NEXT: max.f32 %r7, %r6, %r5;
@@ -13334,6 +13324,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.rn.f16.f32 %rs6, %r10;
; SM70-NEXT: mov.b32 %r11, {%rs6, %rs5};
; SM70-NEXT: cvt.u64.u32 %rd3, %r11;
+; SM70-NEXT: cvt.f32.f16 %r12, %rs8;
; SM70-NEXT: mov.b32 {%rs9, %rs10}, %r34;
; SM70-NEXT: cvt.f32.f16 %r13, %rs10;
; SM70-NEXT: max.f32 %r14, %r13, %r12;
@@ -13360,11 +13351,10 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NEXT: cvt.u32.u64 %r35, %rd12;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r3;
-; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs19, %rs20}, %r4;
-; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
; SM70-NEXT: $L__BB303_3: // %atomicrmw.start2
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.f32.f16 %r19, %rs14;
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r35;
; SM70-NEXT: cvt.f32.f16 %r20, %rs16;
; SM70-NEXT: max.f32 %r21, %r20, %r19;
@@ -13375,6 +13365,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM70-NEXT: cvt.rn.f16.f32 %rs18, %r24;
; SM70-NEXT: mov.b32 %r25, {%rs18, %rs17};
; SM70-NEXT: cvt.u64.u32 %rd13, %r25;
+; SM70-NEXT: cvt.f32.f16 %r26, %rs20;
; SM70-NEXT: mov.b32 {%rs21, %rs22}, %r36;
; SM70-NEXT: cvt.f32.f16 %r27, %rs22;
; SM70-NEXT: max.f32 %r28, %r27, %r26;
@@ -14132,13 +14123,9 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: $L__BB314_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14151,6 +14138,8 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM70-NEXT: or.b32 %r11, %r7, 4194304;
; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
; SM70-NEXT: shl.b32 %r16, %r15, 16;
; SM70-NEXT: add.rn.f32 %r17, %r16, %r14;
@@ -14172,6 +14161,8 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM70-NEXT: or.b32 %r32, %r28, 4194304;
; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
; SM70-NEXT: shl.b32 %r37, %r36, 16;
; SM70-NEXT: add.rn.f32 %r38, %r37, %r35;
@@ -14218,13 +14209,11 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: $L__BB315_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
@@ -14248,6 +14237,8 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
; SM70-NEXT: shl.b32 %r29, %r28, 16;
@@ -14479,13 +14470,9 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: $L__BB318_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14498,6 +14485,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM70-NEXT: or.b32 %r11, %r7, 4194304;
; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
; SM70-NEXT: shl.b32 %r16, %r15, 16;
; SM70-NEXT: sub.rn.f32 %r17, %r16, %r14;
@@ -14519,6 +14508,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM70-NEXT: or.b32 %r32, %r28, 4194304;
; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
; SM70-NEXT: shl.b32 %r37, %r36, 16;
; SM70-NEXT: sub.rn.f32 %r38, %r37, %r35;
@@ -14565,13 +14556,11 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: $L__BB319_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
@@ -14595,6 +14584,8 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
; SM70-NEXT: shl.b32 %r29, %r28, 16;
@@ -14826,13 +14817,9 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: $L__BB322_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -14845,6 +14832,8 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM70-NEXT: or.b32 %r11, %r7, 4194304;
; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
; SM70-NEXT: shl.b32 %r16, %r15, 16;
; SM70-NEXT: min.f32 %r17, %r16, %r14;
@@ -14866,6 +14855,8 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM70-NEXT: or.b32 %r32, %r28, 4194304;
; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
; SM70-NEXT: shl.b32 %r37, %r36, 16;
; SM70-NEXT: min.f32 %r38, %r37, %r35;
@@ -14912,13 +14903,11 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: $L__BB323_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
@@ -14942,6 +14931,8 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
; SM70-NEXT: shl.b32 %r29, %r28, 16;
@@ -15173,13 +15164,9 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT: cvt.u32.u16 %r24, %rs6;
; SM70-NEXT: shl.b32 %r25, %r24, 16;
-; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
-; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: $L__BB326_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r45;
@@ -15192,6 +15179,8 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
; SM70-NEXT: or.b32 %r11, %r7, 4194304;
; SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT: shl.b32 %r14, %r13, 16;
; SM70-NEXT: cvt.u32.u16 %r15, %rs3;
; SM70-NEXT: shl.b32 %r16, %r15, 16;
; SM70-NEXT: max.f32 %r17, %r16, %r14;
@@ -15213,6 +15202,8 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
; SM70-NEXT: or.b32 %r32, %r28, 4194304;
; SM70-NEXT: selp.b32 %r33, %r32, %r31, %p3;
+; SM70-NEXT: cvt.u32.u16 %r34, %rs5;
+; SM70-NEXT: shl.b32 %r35, %r34, 16;
; SM70-NEXT: cvt.u32.u16 %r36, %rs7;
; SM70-NEXT: shl.b32 %r37, %r36, 16;
; SM70-NEXT: max.f32 %r38, %r37, %r35;
@@ -15259,13 +15250,11 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r90}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r89, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
-; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: $L__BB327_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r89;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
@@ -15289,6 +15278,8 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
; SM70-NEXT: prmt.b32 %r25, %r24, %r14, 0x7632U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r25;
+; SM70-NEXT: cvt.u32.u16 %r26, %rs6;
+; SM70-NEXT: shl.b32 %r27, %r26, 16;
; SM70-NEXT: mov.b32 {%rs7, %rs8}, %r90;
; SM70-NEXT: cvt.u32.u16 %r28, %rs8;
; SM70-NEXT: shl.b32 %r29, %r28, 16;
@@ -15524,13 +15515,9 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: cvt.u32.u16 %r12, %rs14;
; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
-; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: $L__BB330_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
@@ -15546,6 +15533,8 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NEXT: setp.eq.b16 %p5, %rs3, -32768;
; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
; SM70-NEXT: setp.lt.f32 %p6, %r10, %r8;
@@ -15569,6 +15558,8 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NEXT: setp.eq.b16 %p13, %rs15, -32768;
; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: cvt.u32.u16 %r18, %rs15;
; SM70-NEXT: shl.b32 %r19, %r18, 16;
; SM70-NEXT: setp.lt.f32 %p14, %r19, %r17;
@@ -15614,16 +15605,14 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
-; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: $L__BB331_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r41;
; SM70-NEXT: setp.eq.b16 %p1, %rs4, -32768;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: setp.lt.f32 %p2, %r8, %r6;
@@ -15649,6 +15638,8 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r42;
; SM70-NEXT: setp.eq.b16 %p9, %rs16, -32768;
; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: setp.lt.f32 %p10, %r17, %r15;
@@ -15887,13 +15878,9 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
-; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
-; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
; SM70-NEXT: cvt.u32.u16 %r12, %rs14;
; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
-; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: $L__BB334_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r21;
@@ -15909,6 +15896,8 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NEXT: setp.eq.b16 %p5, %rs3, 0;
; SM70-NEXT: selp.b16 %rs9, %rs3, %rs1, %p5;
+; SM70-NEXT: cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
; SM70-NEXT: shl.b32 %r10, %r9, 16;
; SM70-NEXT: setp.gt.f32 %p6, %r10, %r8;
@@ -15932,6 +15921,8 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NEXT: setp.eq.b16 %p13, %rs15, 0;
; SM70-NEXT: selp.b16 %rs21, %rs15, %rs13, %p13;
+; SM70-NEXT: cvt.u32.u16 %r16, %rs13;
+; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: cvt.u32.u16 %r18, %rs15;
; SM70-NEXT: shl.b32 %r19, %r18, 16;
; SM70-NEXT: setp.gt.f32 %p14, %r19, %r17;
@@ -15977,16 +15968,14 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r41, %rd2;
; SM70-NEXT: mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
-; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: mov.b32 {%rs13, %rs14}, %r2;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
-; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: $L__BB335_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
; SM70-NEXT: mov.b32 {%rs3, %rs4}, %r41;
; SM70-NEXT: setp.eq.b16 %p1, %rs4, 0;
; SM70-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1;
+; SM70-NEXT: cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT: shl.b32 %r6, %r5, 16;
; SM70-NEXT: cvt.u32.u16 %r7, %rs4;
; SM70-NEXT: shl.b32 %r8, %r7, 16;
; SM70-NEXT: setp.gt.f32 %p2, %r8, %r6;
@@ -16012,6 +16001,8 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM70-NEXT: mov.b32 {%rs15, %rs16}, %r42;
; SM70-NEXT: setp.eq.b16 %p9, %rs16, 0;
; SM70-NEXT: selp.b16 %rs17, %rs16, %rs14, %p9;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs14;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
; SM70-NEXT: cvt.u32.u16 %r16, %rs16;
; SM70-NEXT: shl.b32 %r17, %r16, 16;
; SM70-NEXT: setp.gt.f32 %p10, %r17, %r15;
@@ -16276,31 +16267,29 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB339_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16308,11 +16297,13 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -16505,31 +16496,29 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB343_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16537,11 +16526,13 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -16736,31 +16727,29 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: $L__BB347_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM70-NEXT: cvt.u16.u32 %rs1, %r3;
; SM70-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs2, %r4;
; SM70-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM70-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM70-NEXT: cvt.u16.u32 %rs4, %r6;
; SM70-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs5, %r7;
; SM70-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT: cvt.u32.u16 %r8, %rs6;
; SM70-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM70-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM70-NEXT: cvt.u16.u32 %rs7, %r10;
; SM70-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT: cvt.u16.u32 %rs8, %r11;
; SM70-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM70-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM70-NEXT: cvt.u16.u32 %rs10, %r13;
; SM70-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT: cvt.u16.u32 %rs11, %r14;
; SM70-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -16768,11 +16757,13 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM70-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM70-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs13, %r18;
; SM70-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT: cvt.u16.u32 %rs14, %r19;
; SM70-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM70-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs16, %r21;
; SM70-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -17101,44 +17092,35 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM70-LABEL: nand_monotonic_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r2, %r8, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: ld.param.b16 %r9, [nand_monotonic_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r9, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT: $L__BB361_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
; SM70-NEXT: @%p1 bra $L__BB361_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
@@ -17252,45 +17234,36 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-LABEL: nand_acquire_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r2, %r8, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: ld.param.b16 %r9, [nand_acquire_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r9, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT: $L__BB365_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
; SM70-NEXT: @%p1 bra $L__BB365_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
ret <2 x i8> %retval
@@ -17406,45 +17379,36 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM70-LABEL: nand_release_v2i8_global_cta(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<19>;
+; SM70-NEXT: .reg .b32 %r<17>;
; SM70-NEXT: .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM70-NEXT: mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM70-NEXT: fence.acq_rel.cta;
; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM70-NEXT: and.b32 %r6, %r5, 3;
-; SM70-NEXT: shl.b32 %r1, %r6, 3;
-; SM70-NEXT: mov.b32 %r7, 65535;
-; SM70-NEXT: shl.b32 %r8, %r7, %r1;
-; SM70-NEXT: not.b32 %r2, %r8;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM70-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM70-NEXT: and.b32 %r7, %r6, 3;
+; SM70-NEXT: shl.b32 %r1, %r7, 3;
+; SM70-NEXT: mov.b32 %r8, 65535;
+; SM70-NEXT: shl.b32 %r2, %r8, %r1;
+; SM70-NEXT: not.b32 %r3, %r2;
+; SM70-NEXT: ld.param.b16 %r9, [nand_release_v2i8_global_cta_param_1];
+; SM70-NEXT: shl.b32 %r4, %r9, %r1;
+; SM70-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT: $L__BB369_1: // %atomicrmw.start
; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r9, %r18, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM70-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM70-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM70-NEXT: and.b32 %r11, %r10, %r4;
-; SM70-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM70-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM70-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM70-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM70-NEXT: shl.b32 %r14, %r13, %r1;
-; SM70-NEXT: and.b32 %r15, %r18, %r2;
-; SM70-NEXT: or.b32 %r16, %r15, %r14;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM70-NEXT: mov.b32 %r18, %r3;
+; SM70-NEXT: and.b32 %r10, %r16, %r4;
+; SM70-NEXT: not.b32 %r11, %r10;
+; SM70-NEXT: and.b32 %r12, %r11, %r2;
+; SM70-NEXT: and.b32 %r13, %r16, %r3;
+; SM70-NEXT: or.b32 %r14, %r13, %r12;
+; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM70-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM70-NEXT: mov.b32 %r16, %r5;
; SM70-NEXT: @%p1 bra $L__BB369_1;
; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r17, %r3, %r1;
-; SM70-NEXT: st.param.b16 [func_retval0], %r17;
+; SM70-NEXT: shr.u32 %r15, %r5, %r1;
+; SM70-NEXT: st.param.b16 [func_retval0], %r15;
; SM70-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
index 2e07eaadd297a..2c16672eb9d57 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm90.ll
@@ -10,9 +10,9 @@ define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
; SM90-NEXT: and.b32 %r7, %r6, 3;
@@ -33,7 +33,7 @@ define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r12, %r4, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r12;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r12;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -47,8 +47,8 @@ define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
@@ -70,7 +70,7 @@ define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r12, %r4, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -83,10 +83,10 @@ define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -98,10 +98,10 @@ define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -115,9 +115,9 @@ define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
; SM90-NEXT: and.b32 %r7, %r6, 3;
@@ -138,7 +138,7 @@ define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r12, %r4, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r12;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r12;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -151,10 +151,10 @@ define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -166,10 +166,10 @@ define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -181,15 +181,15 @@ define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i16_global_cta_param_1];
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 amt, dst;
; SM90-NEXT: mov.b128 amt, {%rd2, %rd3};
; SM90-NEXT: atom.acq_rel.cta.global.exch.b128 dst, [%rd1], amt;
; SM90-NEXT: mov.b128 {%rd4, %rd5}, dst;
; SM90-NEXT: }
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -202,12 +202,12 @@ define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -220,13 +220,13 @@ define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [xchg_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -239,15 +239,15 @@ define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -260,10 +260,10 @@ define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r11, [%rd1+8], %r7;
@@ -273,8 +273,8 @@ define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.exch.b32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -286,12 +286,12 @@ define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -303,13 +303,13 @@ define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -321,17 +321,17 @@ define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -343,12 +343,12 @@ define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
@@ -358,10 +358,10 @@ define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xchg ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -375,9 +375,9 @@ define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -400,7 +400,7 @@ define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -415,9 +415,9 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -449,7 +449,7 @@ define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -464,8 +464,8 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
@@ -503,7 +503,7 @@ define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB18_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -518,37 +518,35 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB19_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -556,11 +554,13 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -595,7 +595,7 @@ define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB19_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -609,9 +609,9 @@ define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -634,7 +634,7 @@ define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -648,8 +648,8 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB21_1: // %atomicrmw.start
@@ -661,7 +661,7 @@ define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB21_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -675,8 +675,8 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -700,7 +700,7 @@ define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB22_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -714,8 +714,8 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -769,7 +769,7 @@ define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB23_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -782,12 +782,12 @@ define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -800,13 +800,13 @@ define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -819,15 +819,15 @@ define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -840,10 +840,10 @@ define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -853,8 +853,8 @@ define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -866,12 +866,12 @@ define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -883,13 +883,13 @@ define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -901,17 +901,17 @@ define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -923,12 +923,12 @@ define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
@@ -938,10 +938,10 @@ define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -955,9 +955,9 @@ define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -980,7 +980,7 @@ define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -995,9 +995,9 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -1029,7 +1029,7 @@ define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -1044,8 +1044,8 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
@@ -1083,7 +1083,7 @@ define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB34_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -1098,37 +1098,35 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB35_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: sub.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: sub.s16 %rs12, %rs11, %rs10;
@@ -1136,11 +1134,13 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
; SM90-NEXT: sub.s16 %rs15, %rs14, %rs13;
; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -1175,7 +1175,7 @@ define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB35_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -1189,9 +1189,9 @@ define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -1214,7 +1214,7 @@ define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -1229,8 +1229,8 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
@@ -1246,7 +1246,7 @@ define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB37_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -1261,8 +1261,8 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -1294,7 +1294,7 @@ define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB38_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -1309,8 +1309,8 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -1380,7 +1380,7 @@ define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB39_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -1393,13 +1393,13 @@ define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: neg.s32 %r2, %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r3;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -1412,15 +1412,15 @@ define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: neg.s32 %r3, %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
; SM90-NEXT: neg.s32 %r5, %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r4, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r4, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1433,9 +1433,9 @@ define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: neg.s32 %r5, %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
; SM90-NEXT: neg.s32 %r7, %r2;
@@ -1445,7 +1445,7 @@ define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: neg.s32 %r11, %r4;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -1458,10 +1458,10 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: neg.s32 %r9, %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
; SM90-NEXT: neg.s32 %r11, %r6;
@@ -1479,8 +1479,8 @@ define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: neg.s32 %r23, %r4;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -1492,13 +1492,13 @@ define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<5>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: neg.s64 %rd3, %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd4;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -1510,15 +1510,15 @@ define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<8>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: neg.s64 %rd4, %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
; SM90-NEXT: neg.s64 %rd6, %rd3;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -1530,10 +1530,10 @@ define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<14>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: neg.s64 %rd6, %rd4;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
; SM90-NEXT: neg.s64 %rd8, %rd5;
@@ -1543,8 +1543,8 @@ define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: neg.s64 %rd12, %rd3;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd7, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd11, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd7, %rd9};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -1556,12 +1556,12 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: neg.s64 %rd10, %rd8;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
; SM90-NEXT: neg.s64 %rd12, %rd9;
@@ -1579,10 +1579,10 @@ define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: neg.s64 %rd24, %rd3;
; SM90-NEXT: atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd11, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd23, %rd25};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd19, %rd21};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd15, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd11, %rd13};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise sub ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -1595,9 +1595,9 @@ define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -1610,7 +1610,7 @@ define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
; SM90-NEXT: shr.u32 %r11, %r10, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r11;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r11;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -1623,7 +1623,7 @@ define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
@@ -1632,13 +1632,13 @@ define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: mov.b32 %r4, 65535;
; SM90-NEXT: shl.b32 %r5, %r4, %r3;
; SM90-NEXT: not.b32 %r6, %r5;
-; SM90-NEXT: ld.param.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r7, [and_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: shl.b32 %r8, %r7, %r3;
; SM90-NEXT: or.b32 %r9, %r8, %r6;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
; SM90-NEXT: shr.u32 %r11, %r10, %r3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r11;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -1652,8 +1652,8 @@ define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB50_1: // %atomicrmw.start
@@ -1665,7 +1665,7 @@ define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB50_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -1679,8 +1679,8 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -1704,7 +1704,7 @@ define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB51_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -1717,9 +1717,9 @@ define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -1732,7 +1732,7 @@ define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9;
; SM90-NEXT: shr.u32 %r11, %r10, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r11;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r11;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -1746,8 +1746,8 @@ define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB53_1: // %atomicrmw.start
@@ -1759,7 +1759,7 @@ define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB53_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -1773,8 +1773,8 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -1798,7 +1798,7 @@ define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB54_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -1812,8 +1812,8 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -1867,7 +1867,7 @@ define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB55_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -1880,12 +1880,12 @@ define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -1898,13 +1898,13 @@ define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -1917,15 +1917,15 @@ define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -1938,10 +1938,10 @@ define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
@@ -1951,8 +1951,8 @@ define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -1964,12 +1964,12 @@ define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -1981,13 +1981,13 @@ define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -1999,17 +1999,17 @@ define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -2021,12 +2021,12 @@ define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
@@ -2036,10 +2036,10 @@ define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise and ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -2053,9 +2053,9 @@ define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -2079,7 +2079,7 @@ define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -2089,46 +2089,37 @@ define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-LABEL: nand_acq_rel_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acq_rel_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r2, %r8, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: ld.param::func.b16 %r9, [nand_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r9, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
; SM90-NEXT: @%p1 bra $L__BB65_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -2142,8 +2133,8 @@ define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB66_1: // %atomicrmw.start
@@ -2156,7 +2147,7 @@ define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: @%p1 bra $L__BB66_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -2170,8 +2161,8 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -2197,7 +2188,7 @@ define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: @%p1 bra $L__BB67_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -2211,9 +2202,9 @@ define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -2237,7 +2228,7 @@ define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r15;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -2251,8 +2242,8 @@ define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB69_1: // %atomicrmw.start
@@ -2265,7 +2256,7 @@ define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: @%p1 bra $L__BB69_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -2279,8 +2270,8 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -2306,7 +2297,7 @@ define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: @%p1 bra $L__BB70_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -2320,8 +2311,8 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -2379,7 +2370,7 @@ define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: @%p1 bra $L__BB71_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -2393,9 +2384,9 @@ define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB72_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -2407,7 +2398,7 @@ define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: @%p1 bra $L__BB72_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -2421,8 +2412,8 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<13>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -2447,7 +2438,7 @@ define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: @%p1 bra $L__BB73_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -2461,8 +2452,8 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<28>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -2518,7 +2509,7 @@ define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: @%p1 bra $L__BB74_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -2532,10 +2523,10 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<54>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -2643,8 +2634,8 @@ define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: @%p2 bra $L__BB75_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -2657,9 +2648,9 @@ define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -2671,7 +2662,7 @@ define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: @%p1 bra $L__BB76_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -2684,8 +2675,8 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: .reg .b64 %rd<16>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [nand_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [nand_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [nand_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -2717,7 +2708,7 @@ define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: @%p1 bra $L__BB77_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -2730,10 +2721,10 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: .reg .b64 %rd<30>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v4i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -2793,8 +2784,8 @@ define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: @%p2 bra $L__BB78_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -2807,12 +2798,12 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: .reg .b64 %rd<58>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [nand_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [nand_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [nand_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [nand_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [nand_acq_rel_v8i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -2930,10 +2921,10 @@ define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: @%p4 bra $L__BB79_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -2946,9 +2937,9 @@ define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %va
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -2957,7 +2948,7 @@ define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %va
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -2970,18 +2961,18 @@ define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %va
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
; SM90-NEXT: and.b32 %r2, %r1, 3;
; SM90-NEXT: shl.b32 %r3, %r2, 3;
-; SM90-NEXT: ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: shl.b32 %r5, %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -2995,8 +2986,8 @@ define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %va
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB82_1: // %atomicrmw.start
@@ -3008,7 +2999,7 @@ define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %va
; SM90-NEXT: @%p1 bra $L__BB82_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -3022,8 +3013,8 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -3047,7 +3038,7 @@ define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %va
; SM90-NEXT: @%p1 bra $L__BB83_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -3060,9 +3051,9 @@ define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -3071,7 +3062,7 @@ define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -3085,8 +3076,8 @@ define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB85_1: // %atomicrmw.start
@@ -3098,7 +3089,7 @@ define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB85_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -3112,8 +3103,8 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -3137,7 +3128,7 @@ define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB86_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -3151,8 +3142,8 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -3206,7 +3197,7 @@ define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB87_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -3219,12 +3210,12 @@ define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -3237,13 +3228,13 @@ define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -3256,15 +3247,15 @@ define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -3277,10 +3268,10 @@ define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
@@ -3290,8 +3281,8 @@ define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -3303,12 +3294,12 @@ define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -3320,13 +3311,13 @@ define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -3338,17 +3329,17 @@ define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -3360,12 +3351,12 @@ define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
@@ -3375,10 +3366,10 @@ define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise or ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -3391,9 +3382,9 @@ define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -3402,7 +3393,7 @@ define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -3415,18 +3406,18 @@ define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r1, %rd1;
; SM90-NEXT: and.b32 %r2, %r1, 3;
; SM90-NEXT: shl.b32 %r3, %r2, 3;
-; SM90-NEXT: ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: shl.b32 %r5, %r4, %r3;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -3440,8 +3431,8 @@ define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB98_1: // %atomicrmw.start
@@ -3453,7 +3444,7 @@ define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB98_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -3467,8 +3458,8 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -3492,7 +3483,7 @@ define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB99_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -3505,9 +3496,9 @@ define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd2, %rd1, -4;
; SM90-NEXT: cvt.u32.u64 %r2, %rd1;
; SM90-NEXT: and.b32 %r3, %r2, 3;
@@ -3516,7 +3507,7 @@ define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM90-NEXT: shr.u32 %r7, %r6, %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r7;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r7;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -3530,8 +3521,8 @@ define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB101_1: // %atomicrmw.start
@@ -3543,7 +3534,7 @@ define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB101_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -3557,8 +3548,8 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -3582,7 +3573,7 @@ define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB102_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -3596,8 +3587,8 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -3651,7 +3642,7 @@ define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB103_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -3664,12 +3655,12 @@ define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -3682,13 +3673,13 @@ define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -3701,15 +3692,15 @@ define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -3722,10 +3713,10 @@ define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
@@ -3735,8 +3726,8 @@ define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -3748,12 +3739,12 @@ define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -3765,13 +3756,13 @@ define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -3783,17 +3774,17 @@ define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -3805,12 +3796,12 @@ define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
@@ -3820,10 +3811,10 @@ define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise xor ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -3838,9 +3829,9 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -3866,7 +3857,7 @@ define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -3881,9 +3872,9 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -3920,7 +3911,7 @@ define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r18, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r18;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -3934,22 +3925,22 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM90-NEXT: $L__BB114_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM90-NEXT: setp.gt.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM90-NEXT: setp.gt.s32 %p4, %r10, %r9;
; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -3973,7 +3964,7 @@ define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p5 bra $L__BB114_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -3988,22 +3979,22 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: $L__BB115_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM90-NEXT: setp.gt.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM90-NEXT: setp.gt.s32 %p2, %r6, %r5;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4089,7 +4080,7 @@ define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p9 bra $L__BB115_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r49, %r50};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -4104,9 +4095,9 @@ define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4131,7 +4122,7 @@ define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -4145,8 +4136,8 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB117_1: // %atomicrmw.start
@@ -4158,7 +4149,7 @@ define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB117_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -4172,8 +4163,8 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -4197,7 +4188,7 @@ define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB118_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -4211,8 +4202,8 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -4266,7 +4257,7 @@ define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB119_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -4279,12 +4270,12 @@ define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -4297,13 +4288,13 @@ define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -4316,15 +4307,15 @@ define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -4337,10 +4328,10 @@ define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
@@ -4350,8 +4341,8 @@ define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -4363,12 +4354,12 @@ define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -4380,13 +4371,13 @@ define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -4398,17 +4389,17 @@ define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -4420,12 +4411,12 @@ define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
@@ -4435,10 +4426,10 @@ define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise max ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -4453,9 +4444,9 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4481,7 +4472,7 @@ define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -4496,9 +4487,9 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -4535,7 +4526,7 @@ define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r18, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r18;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r18;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -4549,22 +4540,22 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
-; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM90-NEXT: $L__BB130_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r26, 0, 0x8880U;
; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
; SM90-NEXT: prmt.b32 %r6, %r26, 0, 0x9991U;
; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM90-NEXT: prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM90-NEXT: setp.le.s32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM90-NEXT: prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM90-NEXT: setp.le.s32 %p4, %r10, %r9;
; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
@@ -4588,7 +4579,7 @@ define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p5 bra $L__BB130_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -4603,22 +4594,22 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r50}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r49, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
-; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: $L__BB131_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r49;
; SM90-NEXT: cvt.u64.u32 %rd4, %r50;
; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x8880U;
; SM90-NEXT: prmt.b32 %r4, %r50, 0, 0x8880U;
; SM90-NEXT: setp.le.s32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x9991U;
; SM90-NEXT: prmt.b32 %r6, %r50, 0, 0x9991U;
; SM90-NEXT: setp.le.s32 %p2, %r6, %r5;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0xaaa2U;
@@ -4704,7 +4695,7 @@ define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p9 bra $L__BB131_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r49, %r50};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r49, %r50};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -4719,9 +4710,9 @@ define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -4746,7 +4737,7 @@ define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16>
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -4760,8 +4751,8 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB133_1: // %atomicrmw.start
@@ -4773,7 +4764,7 @@ define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16>
; SM90-NEXT: @%p1 bra $L__BB133_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -4787,8 +4778,8 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -4812,7 +4803,7 @@ define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16>
; SM90-NEXT: @%p1 bra $L__BB134_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -4826,8 +4817,8 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -4881,7 +4872,7 @@ define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16>
; SM90-NEXT: @%p1 bra $L__BB135_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -4894,12 +4885,12 @@ define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -4912,13 +4903,13 @@ define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -4931,15 +4922,15 @@ define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -4952,10 +4943,10 @@ define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
@@ -4965,8 +4956,8 @@ define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -4978,12 +4969,12 @@ define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64>
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -4995,13 +4986,13 @@ define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64>
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -5013,17 +5004,17 @@ define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64>
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -5035,12 +5026,12 @@ define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
@@ -5050,10 +5041,10 @@ define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64>
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise min ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -5068,8 +5059,8 @@ define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -5096,7 +5087,7 @@ define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -5111,9 +5102,9 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -5145,7 +5136,7 @@ define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -5159,22 +5150,22 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM90-NEXT: $L__BB146_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM90-NEXT: setp.gt.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM90-NEXT: setp.gt.u32 %p4, %r10, %r9;
; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5190,7 +5181,7 @@ define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: @%p5 bra $L__BB146_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -5205,24 +5196,22 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: $L__BB147_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM90-NEXT: setp.gt.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM90-NEXT: setp.gt.u32 %p2, %r6, %r5;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5243,7 +5232,9 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM90-NEXT: setp.gt.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -5290,7 +5281,7 @@ define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: @%p9 bra $L__BB147_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -5305,9 +5296,9 @@ define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -5332,7 +5323,7 @@ define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -5346,8 +5337,8 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB149_1: // %atomicrmw.start
@@ -5359,7 +5350,7 @@ define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: @%p1 bra $L__BB149_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -5373,8 +5364,8 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -5398,7 +5389,7 @@ define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: @%p1 bra $L__BB150_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -5412,8 +5403,8 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -5467,7 +5458,7 @@ define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: @%p1 bra $L__BB151_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -5480,12 +5471,12 @@ define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -5498,13 +5489,13 @@ define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -5517,15 +5508,15 @@ define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -5538,10 +5529,10 @@ define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
@@ -5551,8 +5542,8 @@ define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -5564,12 +5555,12 @@ define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -5581,13 +5572,13 @@ define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -5599,17 +5590,17 @@ define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -5621,12 +5612,12 @@ define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
@@ -5636,10 +5627,10 @@ define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umax ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -5654,8 +5645,8 @@ define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -5682,7 +5673,7 @@ define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -5697,9 +5688,9 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -5731,7 +5722,7 @@ define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -5745,22 +5736,22 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM90-NEXT: $L__BB162_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r4, %r18, 0, 0x7770U;
; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
; SM90-NEXT: prmt.b32 %r6, %r18, 0, 0x7771U;
; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r8, %r18, 0, 0x7772U;
; SM90-NEXT: setp.le.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r10, %r18, 0, 0x7773U;
; SM90-NEXT: setp.le.u32 %p4, %r10, %r9;
; SM90-NEXT: selp.b32 %r11, %r10, %r9, %p4;
@@ -5776,7 +5767,7 @@ define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: @%p5 bra $L__BB162_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -5791,24 +5782,22 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: $L__BB163_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM90-NEXT: setp.le.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM90-NEXT: setp.le.u32 %p2, %r6, %r5;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -5829,7 +5818,9 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: prmt.b32 %r17, %r1, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r18, %r33, 0, 0x7773U;
; SM90-NEXT: setp.le.u32 %p8, %r18, %r17;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
+; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: cvt.u16.u32 %rs6, %r8;
@@ -5876,7 +5867,7 @@ define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: @%p9 bra $L__BB163_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -5891,9 +5882,9 @@ define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -5918,7 +5909,7 @@ define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -5932,8 +5923,8 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB165_1: // %atomicrmw.start
@@ -5945,7 +5936,7 @@ define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16
; SM90-NEXT: @%p1 bra $L__BB165_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -5959,8 +5950,8 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -5984,7 +5975,7 @@ define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16
; SM90-NEXT: @%p1 bra $L__BB166_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -5998,8 +5989,8 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -6053,7 +6044,7 @@ define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16
; SM90-NEXT: @%p1 bra $L__BB167_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -6066,12 +6057,12 @@ define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -6084,13 +6075,13 @@ define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -6103,15 +6094,15 @@ define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -6124,10 +6115,10 @@ define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
@@ -6137,8 +6128,8 @@ define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -6150,12 +6141,12 @@ define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -6167,13 +6158,13 @@ define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -6185,17 +6176,17 @@ define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -6207,12 +6198,12 @@ define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
@@ -6222,10 +6213,10 @@ define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise umin ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -6240,8 +6231,8 @@ define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -6271,7 +6262,7 @@ define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -6286,9 +6277,9 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -6327,7 +6318,7 @@ define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -6342,14 +6333,12 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM90-NEXT: $L__BB178_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r3, %r22, 0, 0x7770U;
@@ -6366,7 +6355,9 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: add.s16 %rs8, %rs7, 1;
; SM90-NEXT: setp.ge.u32 %p1, %r3, %r7;
; SM90-NEXT: setp.ge.u32 %p2, %r4, %r8;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM90-NEXT: setp.ge.u32 %p3, %r5, %r9;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM90-NEXT: setp.ge.u32 %p4, %r6, %r10;
; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
; SM90-NEXT: selp.b32 %r12, 0, %r11, %p4;
@@ -6385,7 +6376,7 @@ define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p5 bra $L__BB178_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -6400,14 +6391,12 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM90-NEXT: $L__BB179_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -6438,7 +6427,9 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: prmt.b32 %r10, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs15, %r10;
; SM90-NEXT: add.s16 %rs16, %rs15, 1;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM90-NEXT: setp.ge.u32 %p1, %r3, %r11;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM90-NEXT: setp.ge.u32 %p2, %r4, %r12;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM90-NEXT: setp.ge.u32 %p3, %r5, %r13;
@@ -6485,7 +6476,7 @@ define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p9 bra $L__BB179_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -6500,9 +6491,9 @@ define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -6529,7 +6520,7 @@ define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -6544,8 +6535,8 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r6, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
@@ -6566,7 +6557,7 @@ define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p3 bra $L__BB181_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -6581,8 +6572,8 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r9}, %rd2;
@@ -6623,7 +6614,7 @@ define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p5 bra $L__BB182_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r8, %r9};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r8, %r9};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -6638,8 +6629,8 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -6726,7 +6717,7 @@ define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p9 bra $L__BB183_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r14, %r15, %r16, %r17};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r14, %r15, %r16, %r17};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -6739,12 +6730,12 @@ define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -6757,13 +6748,13 @@ define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -6776,15 +6767,15 @@ define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -6797,10 +6788,10 @@ define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
@@ -6810,8 +6801,8 @@ define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -6824,9 +6815,9 @@ define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM90-NEXT: $L__BB188_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -6839,7 +6830,7 @@ define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p2 bra $L__BB188_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -6852,8 +6843,8 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<16>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -6887,7 +6878,7 @@ define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p3 bra $L__BB189_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -6900,10 +6891,10 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<30>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -6967,8 +6958,8 @@ define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p6 bra $L__BB190_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -6981,12 +6972,12 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<58>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -7112,10 +7103,10 @@ define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p12 bra $L__BB191_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise uinc_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -7130,8 +7121,8 @@ define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -7163,7 +7154,7 @@ define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -7178,9 +7169,9 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -7221,7 +7212,7 @@ define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -7236,14 +7227,12 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r8, %r2, 0, 0x7771U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
-; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM90-NEXT: $L__BB194_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: prmt.b32 %r3, %r26, 0, 0x7770U;
@@ -7264,7 +7253,9 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.eq.b32 %p4, %r6, 0;
; SM90-NEXT: setp.gt.u32 %p5, %r3, %r7;
; SM90-NEXT: setp.gt.u32 %p6, %r4, %r8;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7772U;
; SM90-NEXT: setp.gt.u32 %p7, %r5, %r9;
+; SM90-NEXT: prmt.b32 %r10, %r2, 0, 0x7773U;
; SM90-NEXT: setp.gt.u32 %p8, %r6, %r10;
; SM90-NEXT: cvt.u32.u16 %r11, %rs8;
; SM90-NEXT: selp.b32 %r12, %r10, %r11, %p8;
@@ -7287,7 +7278,7 @@ define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p9 bra $L__BB194_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -7302,16 +7293,12 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
-; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs17, %r11;
-; SM90-NEXT: cvt.u16.u32 %rs18, %r12;
; SM90-NEXT: $L__BB195_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
@@ -7350,7 +7337,9 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.eq.b32 %p6, %r8, 0;
; SM90-NEXT: setp.eq.b32 %p7, %r9, 0;
; SM90-NEXT: setp.eq.b32 %p8, %r10, 0;
+; SM90-NEXT: prmt.b32 %r11, %r2, 0, 0x7770U;
; SM90-NEXT: setp.gt.u32 %p9, %r3, %r11;
+; SM90-NEXT: prmt.b32 %r12, %r2, 0, 0x7771U;
; SM90-NEXT: setp.gt.u32 %p10, %r4, %r12;
; SM90-NEXT: prmt.b32 %r13, %r2, 0, 0x7772U;
; SM90-NEXT: setp.gt.u32 %p11, %r5, %r13;
@@ -7364,6 +7353,8 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: setp.gt.u32 %p15, %r9, %r17;
; SM90-NEXT: prmt.b32 %r18, %r1, 0, 0x7773U;
; SM90-NEXT: setp.gt.u32 %p16, %r10, %r18;
+; SM90-NEXT: cvt.u16.u32 %rs17, %r11;
+; SM90-NEXT: cvt.u16.u32 %rs18, %r12;
; SM90-NEXT: cvt.u16.u32 %rs19, %r13;
; SM90-NEXT: cvt.u16.u32 %rs20, %r14;
; SM90-NEXT: cvt.u16.u32 %rs21, %r15;
@@ -7411,7 +7402,7 @@ define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p17 bra $L__BB195_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -7426,9 +7417,9 @@ define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -7457,7 +7448,7 @@ define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -7472,8 +7463,8 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
@@ -7497,7 +7488,7 @@ define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p5 bra $L__BB197_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -7512,8 +7503,8 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -7561,7 +7552,7 @@ define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p9 bra $L__BB198_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -7576,8 +7567,8 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -7679,7 +7670,7 @@ define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p17 bra $L__BB199_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -7692,12 +7683,12 @@ define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -7710,13 +7701,13 @@ define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -7729,15 +7720,15 @@ define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -7750,10 +7741,10 @@ define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
@@ -7763,8 +7754,8 @@ define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -7777,9 +7768,9 @@ define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<8>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd3];
; SM90-NEXT: $L__BB204_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -7794,7 +7785,7 @@ define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p3 bra $L__BB204_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -7807,8 +7798,8 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -7846,7 +7837,7 @@ define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p5 bra $L__BB205_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -7859,10 +7850,10 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<34>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -7934,8 +7925,8 @@ define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p10 bra $L__BB206_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -7948,12 +7939,12 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<66>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -8095,10 +8086,10 @@ define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p20 bra $L__BB207_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd58, %rd59};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd60, %rd61};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd58, %rd59};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd60, %rd61};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise udec_wrap ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -8113,8 +8104,8 @@ define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -8144,7 +8135,7 @@ define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -8159,9 +8150,9 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -8198,7 +8189,7 @@ define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -8213,16 +8204,12 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
-; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r9;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r7;
; SM90-NEXT: cvt.u16.u32 %rs3, %r5;
; SM90-NEXT: cvt.u16.u32 %rs4, %r3;
; SM90-NEXT: $L__BB210_1: // %atomicrmw.start
@@ -8231,10 +8218,14 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
; SM90-NEXT: prmt.b32 %r6, %r22, 0, 0x7771U;
; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
+; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
; SM90-NEXT: prmt.b32 %r8, %r22, 0, 0x7772U;
; SM90-NEXT: setp.ge.u32 %p3, %r8, %r7;
+; SM90-NEXT: prmt.b32 %r9, %r2, 0, 0x7773U;
; SM90-NEXT: prmt.b32 %r10, %r22, 0, 0x7773U;
; SM90-NEXT: setp.ge.u32 %p4, %r10, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r9;
+; SM90-NEXT: cvt.u16.u32 %rs2, %r7;
; SM90-NEXT: cvt.u16.u32 %rs5, %r4;
; SM90-NEXT: sub.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u16.u32 %rs7, %r6;
@@ -8260,7 +8251,7 @@ define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p5 bra $L__BB210_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -8275,24 +8266,22 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
-; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs13, %r5;
-; SM90-NEXT: cvt.u16.u32 %rs15, %r3;
; SM90-NEXT: $L__BB211_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
; SM90-NEXT: cvt.u64.u32 %rd3, %r33;
; SM90-NEXT: cvt.u64.u32 %rd4, %r34;
; SM90-NEXT: shl.b64 %rd5, %rd4, 32;
; SM90-NEXT: or.b64 %rd6, %rd3, %rd5;
+; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7770U;
; SM90-NEXT: prmt.b32 %r4, %r34, 0, 0x7770U;
; SM90-NEXT: setp.ge.u32 %p1, %r4, %r3;
+; SM90-NEXT: prmt.b32 %r5, %r2, 0, 0x7771U;
; SM90-NEXT: prmt.b32 %r6, %r34, 0, 0x7771U;
; SM90-NEXT: setp.ge.u32 %p2, %r6, %r5;
; SM90-NEXT: prmt.b32 %r7, %r2, 0, 0x7772U;
@@ -8325,7 +8314,9 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: cvt.u16.u32 %rs10, %r10;
; SM90-NEXT: cvt.u16.u32 %rs11, %r7;
; SM90-NEXT: cvt.u16.u32 %rs12, %r8;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r5;
; SM90-NEXT: cvt.u16.u32 %rs14, %r6;
+; SM90-NEXT: cvt.u16.u32 %rs15, %r3;
; SM90-NEXT: cvt.u16.u32 %rs16, %r4;
; SM90-NEXT: sub.s16 %rs17, %rs16, %rs15;
; SM90-NEXT: sub.s16 %rs18, %rs14, %rs13;
@@ -8368,7 +8359,7 @@ define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p9 bra $L__BB211_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -8383,9 +8374,9 @@ define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -8412,7 +8403,7 @@ define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -8427,8 +8418,8 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r2;
@@ -8448,7 +8439,7 @@ define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p3 bra $L__BB213_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -8463,8 +8454,8 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -8504,7 +8495,7 @@ define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p5 bra $L__BB214_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -8519,8 +8510,8 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -8606,7 +8597,7 @@ define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p9 bra $L__BB215_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -8620,9 +8611,9 @@ define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB216_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -8635,7 +8626,7 @@ define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p2 bra $L__BB216_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -8649,8 +8640,8 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -8678,7 +8669,7 @@ define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p3 bra $L__BB217_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -8692,8 +8683,8 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -8755,7 +8746,7 @@ define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p5 bra $L__BB218_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -8769,10 +8760,10 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -8892,8 +8883,8 @@ define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p10 bra $L__BB219_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -8906,9 +8897,9 @@ define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM90-NEXT: $L__BB220_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -8921,7 +8912,7 @@ define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p2 bra $L__BB220_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -8934,8 +8925,8 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<16>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_cond_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -8969,7 +8960,7 @@ define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p3 bra $L__BB221_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -8982,10 +8973,10 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<30>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -9049,8 +9040,8 @@ define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p6 bra $L__BB222_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -9063,12 +9054,12 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<58>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -9194,10 +9185,10 @@ define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p12 bra $L__BB223_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_cond ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -9212,8 +9203,8 @@ define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
@@ -9241,7 +9232,7 @@ define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
ret <1 x i8> %retval
@@ -9256,9 +9247,9 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -9293,7 +9284,7 @@ define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r17, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r17;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
ret <2 x i8> %retval
@@ -9308,8 +9299,8 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
@@ -9351,7 +9342,7 @@ define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: @%p1 bra $L__BB226_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
ret <4 x i8> %retval
@@ -9366,40 +9357,38 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB227_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: max.u16 %rs3, %rs2, %rs1;
; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs4;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs5, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs6, %r7;
; SM90-NEXT: max.u16 %rs7, %rs6, %rs5;
; SM90-NEXT: sub.s16 %rs8, %rs7, %rs5;
; SM90-NEXT: cvt.u32.u16 %r8, %rs8;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs9, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs10, %r11;
; SM90-NEXT: max.u16 %rs11, %rs10, %rs9;
; SM90-NEXT: sub.s16 %rs12, %rs11, %rs9;
; SM90-NEXT: cvt.u32.u16 %r12, %rs12;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs13, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r14;
; SM90-NEXT: max.u16 %rs15, %rs14, %rs13;
@@ -9408,12 +9397,14 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs18, %r19;
; SM90-NEXT: max.u16 %rs19, %rs18, %rs17;
; SM90-NEXT: sub.s16 %rs20, %rs19, %rs17;
; SM90-NEXT: cvt.u32.u16 %r20, %rs20;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs21, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs22, %r22;
@@ -9451,7 +9442,7 @@ define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: @%p1 bra $L__BB227_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
ret <8 x i8> %retval
@@ -9466,9 +9457,9 @@ define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -9494,7 +9485,7 @@ define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
ret <1 x i16> %retval
@@ -9509,8 +9500,8 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r2;
@@ -9527,7 +9518,7 @@ define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB229_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
ret <2 x i16> %retval
@@ -9542,8 +9533,8 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -9577,7 +9568,7 @@ define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p1 bra $L__BB230_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
ret <4 x i16> %retval
@@ -9592,8 +9583,8 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -9667,7 +9658,7 @@ define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p1 bra $L__BB231_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
ret <8 x i16> %retval
@@ -9681,9 +9672,9 @@ define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB232_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -9695,7 +9686,7 @@ define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: @%p1 bra $L__BB232_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
ret <1 x i32> %retval
@@ -9709,8 +9700,8 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -9736,7 +9727,7 @@ define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB233_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
ret <2 x i32> %retval
@@ -9750,8 +9741,8 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -9809,7 +9800,7 @@ define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p1 bra $L__BB234_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
ret <4 x i32> %retval
@@ -9823,10 +9814,10 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -9938,8 +9929,8 @@ define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p2 bra $L__BB235_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
ret <8 x i32> %retval
@@ -9952,9 +9943,9 @@ define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<7>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM90-NEXT: $L__BB236_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -9966,7 +9957,7 @@ define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: @%p1 bra $L__BB236_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
ret <1 x i64> %retval
@@ -9979,8 +9970,8 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<16>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_v2i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [usub_sat_acq_rel_v2i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -10012,7 +10003,7 @@ define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB237_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
ret <2 x i64> %retval
@@ -10025,10 +10016,10 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<30>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -10088,8 +10079,8 @@ define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p2 bra $L__BB238_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
ret <4 x i64> %retval
@@ -10102,12 +10093,12 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<58>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -10225,10 +10216,10 @@ define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p4 bra $L__BB239_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd50, %rd51};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd50, %rd51};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise usub_sat ptr addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
ret <8 x i64> %retval
@@ -10241,12 +10232,12 @@ define <1 x float> @fadd_acq_rel_v1f32_shared_cta(ptr addrspace(3) %addr, <1 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f32_shared_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_v1f32_shared_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_v1f32_shared_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -10259,13 +10250,13 @@ define <2 x float> @fadd_acq_rel_v2f32_shared_cta(ptr addrspace(3) %addr, <2 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f32_shared_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_shared_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_shared_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10278,15 +10269,15 @@ define <4 x float> @fadd_acq_rel_v4f32_shared_cta(ptr addrspace(3) %addr, <4 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f32_shared_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_shared_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_shared_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10299,10 +10290,10 @@ define <8 x float> @fadd_acq_rel_v8f32_shared_cta(ptr addrspace(3) %addr, <8 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f32_shared_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8f32_shared_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_shared_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_shared_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_shared_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_shared_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r11, [%rd1+8], %r7;
@@ -10312,8 +10303,8 @@ define <8 x float> @fadd_acq_rel_v8f32_shared_cta(ptr addrspace(3) %addr, <8 x f
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.shared.add.f32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(3) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -10327,9 +10318,9 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB244_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -10340,7 +10331,7 @@ define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: @%p1 bra $L__BB244_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -10354,8 +10345,8 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -10379,7 +10370,7 @@ define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: @%p1 bra $L__BB245_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10393,8 +10384,8 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -10448,7 +10439,7 @@ define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: @%p1 bra $L__BB246_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10462,10 +10453,10 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -10569,8 +10560,8 @@ define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: @%p2 bra $L__BB247_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -10584,9 +10575,9 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB248_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -10597,7 +10588,7 @@ define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: @%p1 bra $L__BB248_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -10611,8 +10602,8 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -10636,7 +10627,7 @@ define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: @%p1 bra $L__BB249_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10650,8 +10641,8 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -10705,7 +10696,7 @@ define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: @%p1 bra $L__BB250_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10719,10 +10710,10 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -10826,8 +10817,8 @@ define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: @%p2 bra $L__BB251_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -10841,9 +10832,9 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB252_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -10854,7 +10845,7 @@ define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x f
; SM90-NEXT: @%p1 bra $L__BB252_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -10868,8 +10859,8 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -10893,7 +10884,7 @@ define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x f
; SM90-NEXT: @%p1 bra $L__BB253_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -10907,8 +10898,8 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -10962,7 +10953,7 @@ define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x f
; SM90-NEXT: @%p1 bra $L__BB254_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -10976,10 +10967,10 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -11083,8 +11074,8 @@ define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x f
; SM90-NEXT: @%p2 bra $L__BB255_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11098,9 +11089,9 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB256_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -11111,7 +11102,7 @@ define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p1 bra $L__BB256_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -11125,8 +11116,8 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -11150,7 +11141,7 @@ define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p1 bra $L__BB257_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -11164,8 +11155,8 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -11219,7 +11210,7 @@ define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p1 bra $L__BB258_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -11233,10 +11224,10 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -11340,8 +11331,8 @@ define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p2 bra $L__BB259_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11355,9 +11346,9 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB260_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -11368,7 +11359,7 @@ define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: @%p1 bra $L__BB260_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
ret <1 x float> %retval
@@ -11382,8 +11373,8 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -11407,7 +11398,7 @@ define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p1 bra $L__BB261_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
ret <2 x float> %retval
@@ -11421,8 +11412,8 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -11476,7 +11467,7 @@ define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p1 bra $L__BB262_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
ret <4 x float> %retval
@@ -11490,10 +11481,10 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -11597,8 +11588,8 @@ define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p2 bra $L__BB263_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
ret <8 x float> %retval
@@ -11610,12 +11601,12 @@ define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<4>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd3;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -11627,13 +11618,13 @@ define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd4, %rd5};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -11645,17 +11636,17 @@ define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<10>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd8, %rd9};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd6, %rd7};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -11667,12 +11658,12 @@ define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<18>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
@@ -11682,10 +11673,10 @@ define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
; SM90-NEXT: atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd10, %rd11};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd16, %rd17};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd14, %rd15};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd12, %rd13};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd10, %rd11};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -11698,9 +11689,9 @@ define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fsub_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
; SM90-NEXT: $L__BB268_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -11711,7 +11702,7 @@ define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: @%p1 bra $L__BB268_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -11724,8 +11715,8 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<14>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fsub_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -11755,7 +11746,7 @@ define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB269_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -11768,10 +11759,10 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [fsub_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -11827,8 +11818,8 @@ define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p2 bra $L__BB270_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -11841,12 +11832,12 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [fsub_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [fsub_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -11956,10 +11947,10 @@ define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p4 bra $L__BB271_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end21
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd44, %rd45};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -11972,9 +11963,9 @@ define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
; SM90-NEXT: $L__BB272_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -11985,7 +11976,7 @@ define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: @%p1 bra $L__BB272_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -11998,8 +11989,8 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<14>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmin_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fmin_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmin_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -12029,7 +12020,7 @@ define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB273_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -12042,10 +12033,10 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12101,8 +12092,8 @@ define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p2 bra $L__BB274_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12115,12 +12106,12 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [fmin_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12230,10 +12221,10 @@ define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p4 bra $L__BB275_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd44, %rd45};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12246,9 +12237,9 @@ define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<6>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd3];
; SM90-NEXT: $L__BB276_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -12259,7 +12250,7 @@ define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: @%p1 bra $L__BB276_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -12272,8 +12263,8 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<14>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmax_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fmax_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -12303,7 +12294,7 @@ define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB277_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -12316,10 +12307,10 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12375,8 +12366,8 @@ define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p2 bra $L__BB278_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12389,12 +12380,12 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<50>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [fmax_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12504,10 +12495,10 @@ define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p4 bra $L__BB279_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd42, %rd43};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd42, %rd43};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd44, %rd45};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12520,9 +12511,9 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM90-NEXT: $L__BB280_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -12539,7 +12530,7 @@ define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p4 bra $L__BB280_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -12552,8 +12543,8 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<20>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fminimum_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fminimum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -12595,7 +12586,7 @@ define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p7 bra $L__BB281_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -12608,10 +12599,10 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<38>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12691,8 +12682,8 @@ define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p14 bra $L__BB282_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -12705,12 +12696,12 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<74>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [fminimum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -12868,10 +12859,10 @@ define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p28 bra $L__BB283_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd66, %rd67};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd68, %rd69};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd66, %rd67};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd68, %rd69};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -12884,9 +12875,9 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<9>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM90-NEXT: $L__BB284_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -12903,7 +12894,7 @@ define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p4 bra $L__BB284_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b64 [func_retval0], %rd2;
+; SM90-NEXT: st.param::func.b64 [func_retval0], %rd2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
ret <1 x double> %retval
@@ -12916,8 +12907,8 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<20>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b64 {%rd4, %rd5}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_v2f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd4, %rd5}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd3, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd6, 0;
; SM90-NEXT: {
@@ -12959,7 +12950,7 @@ define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p7 bra $L__BB285_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd1, %rd2};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
ret <2 x double> %retval
@@ -12972,10 +12963,10 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<38>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd10, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -13055,8 +13046,8 @@ define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p14 bra $L__BB286_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd7, %rd8};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd5, %rd6};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd7, %rd8};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd5, %rd6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
ret <4 x double> %retval
@@ -13069,12 +13060,12 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: .reg .b64 %rd<74>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
-; SM90-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
-; SM90-NEXT: ld.param.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
-; SM90-NEXT: ld.param.v2.b64 {%rd7, %rd8}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd3, %rd4}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd1, %rd2}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd5, %rd6}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b64 {%rd7, %rd8}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd18, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -13232,10 +13223,10 @@ define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <
; SM90-NEXT: @%p28 bra $L__BB287_7;
; SM90-NEXT: // %bb.8: // %atomicrmw.end19
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b64 [func_retval0+48], {%rd15, %rd16};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+32], {%rd13, %rd14};
-; SM90-NEXT: st.param.v2.b64 [func_retval0], {%rd66, %rd67};
-; SM90-NEXT: st.param.v2.b64 [func_retval0+16], {%rd68, %rd69};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+48], {%rd15, %rd16};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+32], {%rd13, %rd14};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0], {%rd66, %rd67};
+; SM90-NEXT: st.param::func.v2.b64 [func_retval0+16], {%rd68, %rd69};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
ret <8 x double> %retval
@@ -13248,12 +13239,12 @@ define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %rs2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -13266,13 +13257,13 @@ define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -13285,15 +13276,15 @@ define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -13307,9 +13298,9 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
@@ -13327,7 +13318,7 @@ define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -13342,9 +13333,9 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -13369,7 +13360,7 @@ define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -13383,8 +13374,8 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fsub_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB293_1: // %atomicrmw.start
@@ -13396,7 +13387,7 @@ define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: @%p1 bra $L__BB293_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -13410,8 +13401,8 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -13435,7 +13426,7 @@ define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: @%p1 bra $L__BB294_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -13449,8 +13440,8 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -13504,7 +13495,7 @@ define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: @%p1 bra $L__BB295_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -13519,9 +13510,9 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -13546,7 +13537,7 @@ define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -13560,8 +13551,8 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB297_1: // %atomicrmw.start
@@ -13573,7 +13564,7 @@ define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: @%p1 bra $L__BB297_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -13587,8 +13578,8 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -13612,7 +13603,7 @@ define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: @%p1 bra $L__BB298_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -13626,8 +13617,8 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -13681,7 +13672,7 @@ define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: @%p1 bra $L__BB299_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -13696,9 +13687,9 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -13723,7 +13714,7 @@ define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x ha
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -13737,8 +13728,8 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB301_1: // %atomicrmw.start
@@ -13750,7 +13741,7 @@ define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x ha
; SM90-NEXT: @%p1 bra $L__BB301_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -13764,8 +13755,8 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -13789,7 +13780,7 @@ define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x ha
; SM90-NEXT: @%p1 bra $L__BB302_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -13803,8 +13794,8 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -13858,7 +13849,7 @@ define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x ha
; SM90-NEXT: @%p1 bra $L__BB303_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -13873,9 +13864,9 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -13900,7 +13891,7 @@ define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -13914,8 +13905,8 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB305_1: // %atomicrmw.start
@@ -13927,7 +13918,7 @@ define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p1 bra $L__BB305_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -13941,8 +13932,8 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -13966,7 +13957,7 @@ define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p1 bra $L__BB306_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -13980,8 +13971,8 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -14035,7 +14026,7 @@ define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p1 bra $L__BB307_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -14050,9 +14041,9 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -14077,7 +14068,7 @@ define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
ret <1 x half> %retval
@@ -14091,8 +14082,8 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB309_1: // %atomicrmw.start
@@ -14104,7 +14095,7 @@ define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2
; SM90-NEXT: @%p1 bra $L__BB309_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
ret <2 x half> %retval
@@ -14118,8 +14109,8 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -14143,7 +14134,7 @@ define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4
; SM90-NEXT: @%p1 bra $L__BB310_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
ret <4 x half> %retval
@@ -14157,8 +14148,8 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -14212,7 +14203,7 @@ define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8
; SM90-NEXT: @%p1 bra $L__BB311_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
ret <8 x half> %retval
@@ -14225,12 +14216,12 @@ define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %rs2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -14243,13 +14234,13 @@ define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2bf16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs3, [%rd1], %rs1;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs4, [%rd1+2], %rs2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b16 [func_retval0], {%rs3, %rs4};
+; SM90-NEXT: st.param::func.v2.b16 [func_retval0], {%rs3, %rs4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -14262,15 +14253,15 @@ define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4bf16_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs5, [%rd1], %rs1;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs6, [%rd1+2], %rs2;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs7, [%rd1+4], %rs3;
; SM90-NEXT: atom.relaxed.cta.global.add.noftz.bf16 %rs8, [%rd1+6], %rs4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
+; SM90-NEXT: st.param::func.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -14284,9 +14275,9 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
; SM90-NEXT: mov.b32 {%rs1, %rs2}, %r4;
; SM90-NEXT: mov.b32 {%rs3, %rs4}, %r3;
; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r2;
@@ -14304,7 +14295,7 @@ define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: mov.b32 %r6, {%rs11, %rs12};
; SM90-NEXT: mov.b32 %r7, {%rs13, %rs14};
; SM90-NEXT: mov.b32 %r8, {%rs15, %rs16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fadd ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -14319,9 +14310,9 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -14346,7 +14337,7 @@ define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -14360,8 +14351,8 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB317_1: // %atomicrmw.start
@@ -14373,7 +14364,7 @@ define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB317_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -14387,8 +14378,8 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -14412,7 +14403,7 @@ define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p1 bra $L__BB318_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -14426,8 +14417,8 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -14481,7 +14472,7 @@ define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p1 bra $L__BB319_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fsub ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -14496,9 +14487,9 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -14523,7 +14514,7 @@ define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -14537,8 +14528,8 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB321_1: // %atomicrmw.start
@@ -14550,7 +14541,7 @@ define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB321_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -14564,8 +14555,8 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -14589,7 +14580,7 @@ define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p1 bra $L__BB322_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -14603,8 +14594,8 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -14658,7 +14649,7 @@ define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p1 bra $L__BB323_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmin ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -14673,9 +14664,9 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -14700,7 +14691,7 @@ define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -14714,8 +14705,8 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB325_1: // %atomicrmw.start
@@ -14727,7 +14718,7 @@ define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x
; SM90-NEXT: @%p1 bra $L__BB325_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -14741,8 +14732,8 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -14766,7 +14757,7 @@ define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x
; SM90-NEXT: @%p1 bra $L__BB326_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -14780,8 +14771,8 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -14835,7 +14826,7 @@ define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x
; SM90-NEXT: @%p1 bra $L__BB327_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmax ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -14850,9 +14841,9 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -14877,7 +14868,7 @@ define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -14891,8 +14882,8 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB329_1: // %atomicrmw.start
@@ -14904,7 +14895,7 @@ define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB329_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -14918,8 +14909,8 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -14943,7 +14934,7 @@ define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB330_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -14957,8 +14948,8 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -15012,7 +15003,7 @@ define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB331_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fminimum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -15027,9 +15018,9 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
; SM90-NEXT: and.b32 %r5, %r4, 3;
@@ -15054,7 +15045,7 @@ define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r13, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r13;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
ret <1 x bfloat> %retval
@@ -15068,8 +15059,8 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM90-NEXT: $L__BB333_1: // %atomicrmw.start
@@ -15081,7 +15072,7 @@ define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB333_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
ret <2 x bfloat> %retval
@@ -15095,8 +15086,8 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -15120,7 +15111,7 @@ define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB334_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
ret <4 x bfloat> %retval
@@ -15134,8 +15125,8 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: .reg .b64 %rd<26>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -15189,7 +15180,7 @@ define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr,
; SM90-NEXT: @%p1 bra $L__BB335_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise fmaximum ptr addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
ret <8 x bfloat> %retval
@@ -15203,8 +15194,8 @@ define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -15226,7 +15217,7 @@ define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: @%p1 bra $L__BB336_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
ret <1 x i8> %retval
@@ -15241,9 +15232,9 @@ define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: and.b32 %r6, %r5, 3;
@@ -15273,7 +15264,7 @@ define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-NEXT: @%p1 bra $L__BB337_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
@@ -15288,8 +15279,8 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -15325,7 +15316,7 @@ define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: mov.b32 %r18, %r1;
; SM90-NEXT: @%p1 bra $L__BB338_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
ret <4 x i8> %retval
@@ -15340,36 +15331,34 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB339_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -15377,11 +15366,13 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -15415,7 +15406,7 @@ define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
; SM90-NEXT: @%p1 bra $L__BB339_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
ret <8 x i8> %retval
@@ -15429,8 +15420,8 @@ define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -15453,7 +15444,7 @@ define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
ret <1 x i8> %retval
@@ -15468,9 +15459,9 @@ define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
; SM90-NEXT: and.b32 %r6, %r5, 3;
@@ -15501,7 +15492,7 @@ define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
ret <2 x i8> %retval
@@ -15516,8 +15507,8 @@ define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
@@ -15554,7 +15545,7 @@ define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB342_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
ret <4 x i8> %retval
@@ -15569,36 +15560,34 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB343_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -15606,11 +15595,13 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -15645,7 +15636,7 @@ define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB343_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
ret <8 x i8> %retval
@@ -15659,9 +15650,9 @@ define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r6, [add_release_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -15683,7 +15674,7 @@ define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB344_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r14, %r5, %r1;
-; SM90-NEXT: st.param.b8 [func_retval0], %r14;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r14;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
ret <1 x i8> %retval
@@ -15698,9 +15689,9 @@ define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [add_release_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -15731,7 +15722,7 @@ define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %v
; SM90-NEXT: @%p1 bra $L__BB345_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r16, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r16;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r16;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
@@ -15746,8 +15737,8 @@ define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [add_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM90-NEXT: prmt.b32 %r3, %r2, 0, 0x7773U;
@@ -15784,7 +15775,7 @@ define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %v
; SM90-NEXT: mov.b32 %r18, %r1;
; SM90-NEXT: @%p1 bra $L__BB346_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
@@ -15799,37 +15790,35 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r34}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r33, %rd2;
-; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
-; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
-; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
-; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
-; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
-; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
-; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
-; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
-; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
-; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: $L__BB347_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NEXT: prmt.b32 %r3, %r1, 0, 0x7773U;
+; SM90-NEXT: cvt.u16.u32 %rs1, %r3;
; SM90-NEXT: prmt.b32 %r4, %r33, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs2, %r4;
; SM90-NEXT: add.s16 %rs3, %rs2, %rs1;
; SM90-NEXT: cvt.u32.u16 %r5, %rs3;
+; SM90-NEXT: prmt.b32 %r6, %r1, 0, 0x7772U;
+; SM90-NEXT: cvt.u16.u32 %rs4, %r6;
; SM90-NEXT: prmt.b32 %r7, %r33, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs5, %r7;
; SM90-NEXT: add.s16 %rs6, %rs5, %rs4;
; SM90-NEXT: cvt.u32.u16 %r8, %rs6;
; SM90-NEXT: prmt.b32 %r9, %r8, %r5, 0x3340U;
+; SM90-NEXT: prmt.b32 %r10, %r1, 0, 0x7771U;
+; SM90-NEXT: cvt.u16.u32 %rs7, %r10;
; SM90-NEXT: prmt.b32 %r11, %r33, 0, 0x7771U;
; SM90-NEXT: cvt.u16.u32 %rs8, %r11;
; SM90-NEXT: add.s16 %rs9, %rs8, %rs7;
; SM90-NEXT: cvt.u32.u16 %r12, %rs9;
+; SM90-NEXT: prmt.b32 %r13, %r1, 0, 0x7770U;
+; SM90-NEXT: cvt.u16.u32 %rs10, %r13;
; SM90-NEXT: prmt.b32 %r14, %r33, 0, 0x7770U;
; SM90-NEXT: cvt.u16.u32 %rs11, %r14;
; SM90-NEXT: add.s16 %rs12, %rs11, %rs10;
@@ -15837,11 +15826,13 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM90-NEXT: prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM90-NEXT: cvt.u64.u32 %rd3, %r17;
+; SM90-NEXT: prmt.b32 %r18, %r2, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs13, %r18;
; SM90-NEXT: prmt.b32 %r19, %r34, 0, 0x7773U;
; SM90-NEXT: cvt.u16.u32 %rs14, %r19;
; SM90-NEXT: add.s16 %rs15, %rs14, %rs13;
; SM90-NEXT: cvt.u32.u16 %r20, %rs15;
+; SM90-NEXT: prmt.b32 %r21, %r2, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs16, %r21;
; SM90-NEXT: prmt.b32 %r22, %r34, 0, 0x7772U;
; SM90-NEXT: cvt.u16.u32 %rs17, %r22;
@@ -15875,7 +15866,7 @@ define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %v
; SM90-NEXT: cvt.u32.u64 %r33, %rd11;
; SM90-NEXT: @%p1 bra $L__BB347_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r33, %r34};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r33, %r34};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
@@ -15892,10 +15883,10 @@ define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i3
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
ret <1 x i32> %retval
@@ -15908,11 +15899,11 @@ define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i3
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
ret <2 x i32> %retval
@@ -15925,13 +15916,13 @@ define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i3
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
ret <4 x i32> %retval
@@ -15944,9 +15935,9 @@ define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i3
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -15955,8 +15946,8 @@ define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i3
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
ret <8 x i32> %retval
@@ -15969,11 +15960,11 @@ define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
ret <1 x i32> %retval
@@ -15986,12 +15977,12 @@ define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
ret <2 x i32> %retval
@@ -16004,14 +15995,14 @@ define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
ret <4 x i32> %retval
@@ -16024,9 +16015,9 @@ define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -16036,8 +16027,8 @@ define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
ret <8 x i32> %retval
@@ -16050,11 +16041,11 @@ define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [add_release_v1i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
ret <1 x i32> %retval
@@ -16067,12 +16058,12 @@ define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r3, %r4};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r3, %r4};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
ret <2 x i32> %retval
@@ -16085,14 +16076,14 @@ define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
ret <4 x i32> %retval
@@ -16105,10 +16096,10 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [add_release_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
@@ -16117,8 +16108,8 @@ define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32>
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise add ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
@@ -16136,8 +16127,8 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -16160,7 +16151,7 @@ define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8>
; SM90-NEXT: @%p1 bra $L__BB360_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
-; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
ret <1 x i8> %retval
@@ -16170,44 +16161,35 @@ define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8>
; SM90-LABEL: nand_monotonic_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_monotonic_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r2, %r8, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: ld.param::func.b16 %r9, [nand_monotonic_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r9, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM90-NEXT: $L__BB361_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
; SM90-NEXT: @%p1 bra $L__BB361_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
ret <2 x i8> %retval
@@ -16221,8 +16203,8 @@ define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB362_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -16233,7 +16215,7 @@ define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8>
; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB362_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
ret <4 x i8> %retval
@@ -16247,8 +16229,8 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
@@ -16272,7 +16254,7 @@ define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8>
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
; SM90-NEXT: @%p1 bra $L__BB363_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
ret <8 x i8> %retval
@@ -16286,8 +16268,8 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
-; SM90-NEXT: ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -16311,7 +16293,7 @@ define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
ret <1 x i8> %retval
@@ -16321,45 +16303,36 @@ define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-LABEL: nand_acquire_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_acquire_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r2, %r8, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: ld.param::func.b16 %r9, [nand_acquire_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r9, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM90-NEXT: $L__BB365_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
; SM90-NEXT: @%p1 bra $L__BB365_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
ret <2 x i8> %retval
@@ -16373,8 +16346,8 @@ define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB366_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -16386,7 +16359,7 @@ define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: @%p1 bra $L__BB366_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
ret <4 x i8> %retval
@@ -16400,8 +16373,8 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
@@ -16426,7 +16399,7 @@ define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: @%p1 bra $L__BB367_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
ret <8 x i8> %retval
@@ -16440,9 +16413,9 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
; SM90-NEXT: cvt.u32.u64 %r7, %rd2;
; SM90-NEXT: and.b32 %r8, %r7, 3;
@@ -16465,7 +16438,7 @@ define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %
; SM90-NEXT: @%p1 bra $L__BB368_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: shr.u32 %r15, %r5, %r1;
-; SM90-NEXT: st.param.b8 [func_retval0], %r15;
+; SM90-NEXT: st.param::func.b8 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i8> %val syncscope("block") release
ret <1 x i8> %retval
@@ -16475,45 +16448,36 @@ define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %
; SM90-LABEL: nand_release_v2i8_global_cta(
; SM90: {
; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<9>;
-; SM90-NEXT: .reg .b32 %r<19>;
+; SM90-NEXT: .reg .b32 %r<17>;
; SM90-NEXT: .reg .b64 %rd<3>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [nand_release_v2i8_global_cta_param_1];
-; SM90-NEXT: mov.b32 %r4, {%rs1, %rs2};
-; SM90-NEXT: ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
-; SM90-NEXT: and.b32 %r6, %r5, 3;
-; SM90-NEXT: shl.b32 %r1, %r6, 3;
-; SM90-NEXT: mov.b32 %r7, 65535;
-; SM90-NEXT: shl.b32 %r8, %r7, %r1;
-; SM90-NEXT: not.b32 %r2, %r8;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r18, [%rd1];
+; SM90-NEXT: cvt.u32.u64 %r6, %rd2;
+; SM90-NEXT: and.b32 %r7, %r6, 3;
+; SM90-NEXT: shl.b32 %r1, %r7, 3;
+; SM90-NEXT: mov.b32 %r8, 65535;
+; SM90-NEXT: shl.b32 %r2, %r8, %r1;
+; SM90-NEXT: not.b32 %r3, %r2;
+; SM90-NEXT: ld.param::func.b16 %r9, [nand_release_v2i8_global_cta_param_1];
+; SM90-NEXT: shl.b32 %r4, %r9, %r1;
+; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM90-NEXT: $L__BB369_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r9, %r18, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs3, %r9;
-; SM90-NEXT: shr.u16 %rs4, %rs3, 8;
-; SM90-NEXT: mov.b32 %r10, {%rs3, %rs4};
-; SM90-NEXT: and.b32 %r11, %r10, %r4;
-; SM90-NEXT: xor.b32 %r12, %r11, 16711935;
-; SM90-NEXT: mov.b32 {%rs5, %rs6}, %r12;
-; SM90-NEXT: shl.b16 %rs7, %rs6, 8;
-; SM90-NEXT: or.b16 %rs8, %rs5, %rs7;
-; SM90-NEXT: cvt.u32.u16 %r13, %rs8;
-; SM90-NEXT: shl.b32 %r14, %r13, %r1;
-; SM90-NEXT: and.b32 %r15, %r18, %r2;
-; SM90-NEXT: or.b32 %r16, %r15, %r14;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r18, %r16;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r18;
-; SM90-NEXT: mov.b32 %r18, %r3;
+; SM90-NEXT: and.b32 %r10, %r16, %r4;
+; SM90-NEXT: not.b32 %r11, %r10;
+; SM90-NEXT: and.b32 %r12, %r11, %r2;
+; SM90-NEXT: and.b32 %r13, %r16, %r3;
+; SM90-NEXT: or.b32 %r14, %r13, %r12;
+; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14;
+; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16;
+; SM90-NEXT: mov.b32 %r16, %r5;
; SM90-NEXT: @%p1 bra $L__BB369_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r17, %r3, %r1;
-; SM90-NEXT: st.param.b16 [func_retval0], %r17;
+; SM90-NEXT: shr.u32 %r15, %r5, %r1;
+; SM90-NEXT: st.param::func.b16 [func_retval0], %r15;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i8> %val syncscope("block") release
ret <2 x i8> %retval
@@ -16527,8 +16491,8 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r2, [nand_release_v4i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB370_1: // %atomicrmw.start
@@ -16540,7 +16504,7 @@ define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %
; SM90-NEXT: mov.b32 %r5, %r1;
; SM90-NEXT: @%p1 bra $L__BB370_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r1;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i8> %val syncscope("block") release
ret <4 x i8> %retval
@@ -16554,8 +16518,8 @@ define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: .reg .b64 %rd<12>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r8}, %rd2;
@@ -16580,7 +16544,7 @@ define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %
; SM90-NEXT: cvt.u32.u64 %r7, %rd11;
; SM90-NEXT: @%p1 bra $L__BB371_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r7, %r8};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r7, %r8};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i8> %val syncscope("block") release
ret <8 x i8> %retval
@@ -16598,8 +16562,8 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB372_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -16610,7 +16574,7 @@ define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i
; SM90-NEXT: mov.b32 %r5, %r2;
; SM90-NEXT: @%p1 bra $L__BB372_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
ret <1 x i32> %retval
@@ -16624,8 +16588,8 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: .reg .b64 %rd<13>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -16648,7 +16612,7 @@ define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
; SM90-NEXT: @%p1 bra $L__BB373_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
ret <2 x i32> %retval
@@ -16662,8 +16626,8 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: .reg .b64 %rd<28>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -16717,7 +16681,7 @@ define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
; SM90-NEXT: @%p1 bra $L__BB374_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
ret <4 x i32> %retval
@@ -16731,9 +16695,9 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: .reg .b64 %rd<54>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -16840,8 +16804,8 @@ define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
; SM90-NEXT: @%p2 bra $L__BB375_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
ret <8 x i32> %retval
@@ -16855,8 +16819,8 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB376_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -16868,7 +16832,7 @@ define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: @%p1 bra $L__BB376_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
ret <1 x i32> %retval
@@ -16882,8 +16846,8 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<13>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
; SM90-NEXT: cvt.u32.u64 %r5, %rd2;
@@ -16907,7 +16871,7 @@ define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: @%p1 bra $L__BB377_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
ret <2 x i32> %retval
@@ -16921,8 +16885,8 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<28>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -16977,7 +16941,7 @@ define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: @%p1 bra $L__BB378_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
ret <4 x i32> %retval
@@ -16991,9 +16955,9 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<54>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -17101,8 +17065,8 @@ define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: @%p2 bra $L__BB379_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
ret <8 x i32> %retval
@@ -17116,9 +17080,9 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: .reg .b64 %rd<2>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM90-NEXT: $L__BB380_1: // %atomicrmw.start
; SM90-NEXT: // =>This Inner Loop Header: Depth=1
@@ -17129,7 +17093,7 @@ define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32
; SM90-NEXT: mov.b32 %r5, %r2;
; SM90-NEXT: @%p1 bra $L__BB380_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <1 x i32> %val syncscope("block") release
ret <1 x i32> %retval
@@ -17143,8 +17107,8 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: .reg .b64 %rd<13>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM90-NEXT: mov.b64 {_, %r6}, %rd2;
@@ -17168,7 +17132,7 @@ define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32
; SM90-NEXT: cvt.u32.u64 %r5, %rd12;
; SM90-NEXT: @%p1 bra $L__BB381_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v2.b32 [func_retval0], {%r5, %r6};
+; SM90-NEXT: st.param::func.v2.b32 [func_retval0], {%r5, %r6};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <2 x i32> %val syncscope("block") release
ret <2 x i32> %retval
@@ -17182,8 +17146,8 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: .reg .b64 %rd<28>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
@@ -17238,7 +17202,7 @@ define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32
; SM90-NEXT: setp.ne.b64 %p1, %rd27, 0;
; SM90-NEXT: @%p1 bra $L__BB382_1;
; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <4 x i32> %val syncscope("block") release
ret <4 x i32> %retval
@@ -17252,10 +17216,10 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: .reg .b64 %rd<54>;
; SM90-EMPTY:
; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
+; SM90-NEXT: ld.param::func.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1];
-; SM90-NEXT: ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1+16];
+; SM90-NEXT: ld.param::func.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1];
+; SM90-NEXT: ld.param::func.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1+16];
; SM90-NEXT: mov.b64 %rd2, 0;
; SM90-NEXT: {
; SM90-NEXT: .reg .b128 cmp, swap, dst;
@@ -17362,8 +17326,8 @@ define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32
; SM90-NEXT: setp.ne.b64 %p2, %rd53, 0;
; SM90-NEXT: @%p2 bra $L__BB383_3;
; SM90-NEXT: // %bb.4: // %atomicrmw.end1
-; SM90-NEXT: st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
-; SM90-NEXT: st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
+; SM90-NEXT: st.param::func.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM90-NEXT: ret;
%retval = atomicrmw elementwise nand ptr addrspace(1) %addr, <8 x i32> %val syncscope("block") release
ret <8 x i32> %retval
>From ff7cc6055e6111eb37426855ad52813f0ef4cac4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 2 Oct 2026 06:20:51 +0000
Subject: [PATCH 15/15] [NVPTX] Fix elementwise xchg and fence splitting
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 14 ++++++++---
.../NVPTX/expand-atomic-rmw-elementwise.ll | 24 ++++++++++++++++++-
2 files changed, 34 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 8730e68f2c5fb..079dc483429b9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7774,8 +7774,16 @@ getScalarAtomicRMWExpansion(const AtomicRMWInst *AI, Type *Ty,
if (Ty->isVectorTy())
return AtomicExpansionKind::CmpXChg;
- assert(Ty->isIntegerTy() && "Ty should be integer at this point");
- const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
+ // AtomicExpand casts non-integer xchg operations to the same-width integer
+ // type. When classifying an elementwise xchg, this hook runs before that
+ // cast is performed on the scalar lanes, so account for it here.
+ assert(
+ (Ty->isIntegerTy() || AI->getOperation() == AtomicRMWInst::BinOp::Xchg) &&
+ "Ty should be integer at this point");
+ const unsigned BitWidth =
+ Ty->isIntegerTy()
+ ? cast<IntegerType>(Ty)->getBitWidth()
+ : AI->getDataLayout().getTypeStoreSizeInBits(Ty).getFixedValue();
switch (AI->getOperation()) {
default:
@@ -7950,7 +7958,7 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent) {
AtomicExpansionKind ExpansionKind = shouldExpandAtomicRMWInIR(RI);
if (ExpansionKind == AtomicExpansionKind::None ||
- ExpansionKind == AtomicExpansionKind::Expand)
+ (ExpansionKind == AtomicExpansionKind::Expand && !RI->isElementwise()))
return AtomicOrdering::Acquire;
}
diff --git a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
index cb8b7a7a54af1..eb9cfe5442a52 100644
--- a/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
+++ b/llvm/test/Transforms/AtomicExpand/NVPTX/expand-atomic-rmw-elementwise.ll
@@ -143,6 +143,29 @@ entry:
ret <4 x i8> %old
}
+; Non-integer xchg lanes are classified using the integer type to which they
+; will be bitcast, then expanded into native scalar xchg operations.
+define <2 x float> @xchg_v2f32_elementwise(ptr %addr, <2 x float> %val) {
+; CHECK-LABEL: @xchg_v2f32_elementwise(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LO_VAL:%.*]] = extractelement <2 x float> [[VAL:%.*]], i64 0
+; CHECK-NEXT: [[HI_VAL:%.*]] = extractelement <2 x float> [[VAL]], i64 1
+; CHECK-NEXT: [[HI_PTR:%.*]] = getelementptr inbounds float, ptr [[ADDR:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float [[LO_VAL]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = atomicrmw xchg ptr [[ADDR]], i32 [[TMP0]] monotonic, align 8
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[HI_VAL]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = atomicrmw xchg ptr [[HI_PTR]], i32 [[TMP3]] monotonic, align 4
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP4]] to float
+; CHECK-NEXT: [[LO_OLD:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
+; CHECK-NEXT: [[HI_OLD:%.*]] = insertelement <2 x float> [[LO_OLD]], float [[TMP5]], i64 1
+; CHECK-NEXT: ret <2 x float> [[HI_OLD]]
+;
+entry:
+ %old = atomicrmw elementwise xchg ptr %addr, <2 x float> %val monotonic
+ ret <2 x float> %old
+}
+
; Partword bitwise vector atomics must bitcast before widening; zext directly
; from <2 x i8> to i32 is invalid IR.
define <2 x i8> @and_v2i8_partword_elementwise(ptr %addr, <2 x i8> %val) {
@@ -258,4 +281,3 @@ entry:
!0 = !{!1, !2}
!1 = !{!"foo", !"bar"}
!2 = !{!"bux", !"baz"}
-
More information about the llvm-commits
mailing list